Skip to content

Instantly share code, notes, and snippets.

@jack-nl

jack-nl/master1.log Secret

Created May 6, 2021 11:59
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jack-nl/972e202e6e1d31b6a7fe69159b0c4d72 to your computer and use it in GitHub Desktop.
Save jack-nl/972e202e6e1d31b6a7fe69159b0c4d72 to your computer and use it in GitHub Desktop.
Download ZIP
master1.log
Raw
master1.log
[2021-05-03T13:54:46,922][INFO ][o.e.c.s.ClusterApplierService] [master1] removed {{master2}{RBMXwoyITeqpwJlY7VFFhA}{h3z8mbPFRhKjI0caC1rtZg}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}}, term: 38, version: 2131725, reason: ApplyCommitRequest{term=38, version=2131725, sourceNode={master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}
[2021-05-03T14:11:07,057][INFO ][o.e.c.s.ClusterApplierService] [master1] added {{master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}}, term: 38, version: 2131737, reason: ApplyCommitRequest{term=38, version=2131737, sourceNode={master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}
[2021-05-03T14:29:07,886][INFO ][o.e.c.c.Coordinator ] [master1] master node [{master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}] failed, restarting discovery
org.elasticsearch.transport.NodeDisconnectedException: [master0][192.168.0.10:9300][disconnected] disconnected
[2021-05-03T14:29:07,890][INFO ][o.e.c.s.ClusterApplierService] [master1] master node changed {previous [{master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}], current []}, term: 38, version: 2131738, reason: becoming candidate: onLeaderFailure
[2021-05-03T14:29:07,898][WARN ][o.e.c.NodeConnectionsService] [master1] failed to connect to {master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} (tried [1] times)
org.elasticsearch.transport.ConnectTransportException: [master0][192.168.0.10:9300] connect_exception
at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.5.2.jar:7.5.2]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]
at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 192.168.0.10/192.168.0.10:9300
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]
... 7 more
[2021-05-03T14:29:08,305][INFO ][o.e.c.c.JoinHelper ] [master1] failed to join {master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true} with JoinRequest{sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, optionalJoin=Optional[Join{term=39, lastAcceptedTerm=38, lastAcceptedVersion=2131738, sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, targetNode={master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}}]}
org.elasticsearch.transport.RemoteTransportException: [master2][192.168.0.12:9300][internal:cluster/coordination/join]
Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 39 while handling publication
at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1049) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:268) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T14:29:08,430][INFO ][o.e.c.s.MasterService ] [master1] elected-as-master ([2] nodes joined)[{master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true} elect leader, {master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 40, version: 2131739, delta: master node changed {previous [], current [{master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}]}
[2021-05-03T14:29:17,615][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,615][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,616][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,617][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,617][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,616][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,615][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:17,630][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:18,470][WARN ][o.e.g.IncrementalClusterStateWriter] [master1] writing cluster state took [10005ms] which is above the warn threshold of [10s]; wrote metadata for [4361] indices and skipped [0] unchanged indices
[2021-05-03T14:29:18,473][INFO ][o.e.c.c.C.CoordinatorPublication] [master1] after [10s] publication of cluster state version [2131739] is still waiting for {master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true} [WAITING_FOR_QUORUM], {master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true} [SENT_PUBLISH_REQUEST], {data17b}{7uaz7rwQRg-UQQ4WLhclBg}{SXb-eNTSTP-9-dO7pgnEfQ}{192.168.0.37}{192.168.0.37:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data16a}{SzyPNxe6Rm6H9ucP26H6Mw}{PALqrfCdR6CAvYfrioYAZQ}{192.168.0.36}{192.168.0.36:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data06}{NqAsywRoTVSuqTxD8cmq3A}{ffiDh9EhT0OpZ0Suj4hHcQ}{192.168.0.26}{192.168.0.26:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data12a}{22tiK4-STyC2XRJ01nWQ3A}{rKiBCqOFTGKTBesQCzI7aw}{192.168.0.32}{192.168.0.32:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data08}{GIaGadJ0RhyBUePNzWpBdA}{fz0VawBhT_6hIre0uymIfw}{192.168.0.28}{192.168.0.28:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data07}{uYjW78V_QAaAQPHjUSNjjQ}{ZA9njwKuRgOEdIllyccwqQ}{192.168.0.27}{192.168.0.27:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data18a}{_MeUtbWCQK6WAcOWs6L4fQ}{qjYEjHO0Qp-pysLXoM8iYA}{192.168.0.38}{192.168.0.38:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data04}{M7yBnQeXTwCohL8IWN4PBg}{HRi5FXR0S_m2YlWM5zAMCw}{192.168.0.24}{192.168.0.24:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data10b}{WBnLOtGgS6KXAV8FsiFYjg}{D1xTZOswTAiMc3h1YMae3Q}{192.168.0.30}{192.168.0.30:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data18b}{5HTiO0Z8RM2tRi0kaNQQ8w}{O-T8IWgXQIekz9QHQyKy4g}{192.168.0.38}{192.168.0.38:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data05}{3sy0vv41TXGy4Yt8ywpXVw}{AoblOEIlSF2ZWQSqEERqwA}{192.168.0.25}{192.168.0.25:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data10a}{Cl-VWQCjQ3OStxiPmdIXrQ}{KjVwOpXuQCKLy2GQFBluZg}{192.168.0.30}{192.168.0.30:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data15a}{0sFGQdJwSNermm8lSE7v6w}{TU-BKgdwQO2xOnhMx6o-ow}{192.168.0.35}{192.168.0.35:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data11a}{EI9MEUGwQP6yuwlmVsH-EA}{QRz_oL35TyOvnBqTWqWgEA}{192.168.0.31}{192.168.0.31:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data11b}{6FVMYvwiQD-W6fpHtVXPfg}{dAQSQFZMSjKq6kxq8cc98g}{192.168.0.31}{192.168.0.31:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data15b}{bT-KQ6TgQHWYl3pLMjj9MQ}{I9wc6TOxQ2WkqbOZ89g8XQ}{192.168.0.35}{192.168.0.35:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data17a}{x-YWJyTxRoCh8Lf4oS0qWQ}{MqzCX5bVRCa7MWZ3e7CtZg}{192.168.0.37}{192.168.0.37:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data09}{dN08oWhmSNObQgeb9xH_-A}{9aLNT59cQZutpH2xd1LzTQ}{192.168.0.29}{192.168.0.29:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data13a}{vF-2rZ1ZTuG8rcYDTScDbQ}{NY6x69ejT9ehohT-uXKyww}{192.168.0.33}{192.168.0.33:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data03}{9MyOkgP3TuSevNQJkEmOnw}{MBDXNqQOR6G05JiS0UQlEw}{192.168.0.23}{192.168.0.23:9300}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data16b}{2afokmtOQGGzfFLz3v4_Bg}{Lo4izLlTQqSMfbrf21aAkw}{192.168.0.36}{192.168.0.36:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data13b}{mAQ5jgAMQjaYzwpVE83kcg}{LBsGSuLJR5Kg8iWCA-SX1A}{192.168.0.33}{192.168.0.33:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM], {data12b}{Fs5fJqYNSLOPK7OVW7qaYQ}{2SeThcYbQ5uUG6c7JyP_Qg}{192.168.0.32}{192.168.0.32:9301}{di}{xpack.installed=true} [WAITING_FOR_QUORUM]
[2021-05-03T14:29:27,616][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,618][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,618][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,618][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,618][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,619][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,623][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:27,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,618][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,618][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,621][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,622][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,623][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,623][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,623][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:37,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:38,476][WARN ][o.e.c.s.MasterService ] [master1] failing [elected-as-master ([2] nodes joined)[{master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true} elect leader, {master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_]]: failed to commit cluster state version [2131739]
org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1429) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:88) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1349) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:89) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$2.run(Coordinator.java:1302) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: timed out after 30s
at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:86) ~[elasticsearch-7.5.2.jar:7.5.2]
... 5 more
[2021-05-03T14:29:38,477][INFO ][o.e.c.c.JoinHelper ] [master1] failed to join {master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true} with JoinRequest{sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, optionalJoin=Optional[Join{term=40, lastAcceptedTerm=38, lastAcceptedVersion=2131738, sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, targetNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{GyE6pKvWTfeRglHhD-5oCw}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}}]}
org.elasticsearch.transport.RemoteTransportException: [master1][192.168.0.11:9300][internal:cluster/coordination/join]
Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1429) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:88) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1349) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:89) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$2.run(Coordinator.java:1302) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: timed out after 30s
at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:86) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$2.run(Coordinator.java:1302) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:830) ~[?:?]
[2021-05-03T14:29:47,626][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:47,628][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:47,630][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] timed out while retrying [cluster:monitor/state] after failure (timeout [30s])
[2021-05-03T14:29:47,630][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] timed out while retrying [cluster:monitor/state] after failure (timeout [30s])
[2021-05-03T14:29:47,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:47,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] timed out while retrying [cluster:monitor/state] after failure (timeout [30s])
[2021-05-03T14:29:47,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] timed out while retrying [cluster:monitor/state] after failure (timeout [30s])
[2021-05-03T14:29:47,631][WARN ][r.suppressed ] [master1] path: /_cluster/state/master_node, params: {metric=master_node, settings_filter=xpack.security.authc.realms.oidc.*.ssl.keystore.path,transport.profiles.*.xpack.security.ssl.trust_restrictions,reindex.ssl.supported_protocols,xpack.security.authc.realms.ldap.*.ssl.verification_mode,xpack.security.authc.realms.ldap.*.ssl.keystore.key_password,xpack.security.http.ssl.truststore.password,xpack.security.http.ssl.keystore.password,xpack.notification.email.ssl.supported_protocols,xpack.security.authc.realms.ldap.*.ssl.truststore.path,xpack.security.transport.ssl.truststore.password,xpack.notification.email.ssl.truststore.password,xpack.security.authc.realms.active_directory.*.ssl.verification_mode,xpack.http.ssl.truststore.algorithm,xpack.security.http.ssl.supported_protocols,reindex.ssl.truststore.password,xpack.security.authc.realms.oidc.*.ssl.verification_mode,xpack.security.transport.ssl.supported_protocols,xpack.security.authc.realms.saml.*.ssl.keystore.path,xpack.security.http.ssl.truststore.algorithm,xpack.security.transport.ssl.keystore.key_password,xpack.security.http.ssl.key,xpack.security.authc.realms.saml.*.encryption.key,xpack.security.http.ssl.keystore.type,xpack.security.http.ssl.verification_mode,xpack.security.authc.realms.pki.*.truststore.path,xpack.security.authc.realms.saml.*.signing.keystore.path,xpack.security.authc.realms.ldap.*.ssl.key,xpack.security.authc.realms.oidc.*.ssl.keystore.password,transport.profiles.*.xpack.security.ssl.supported_protocols,reindex.ssl.keystore.key_password,xpack.security.authc.realms.active_directory.*.ssl.supported_protocols,xpack.security.authc.realms.active_directory.*.hostname_verification,xpack.security.authc.realms.saml.*.ssl.truststore.algorithm,xpack.security.authc.realms.ldap.*.ssl.certificate_authorities,xpack.security.authc.realms.saml.*.signing.key,xpack.security.authc.realms.oidc.*.ssl.truststore.type,xpack.security.authc.realms.active_directory.*.ssl.keystore.key_password,transport.profiles.*.xpack.security.ssl.truststore.path,xpack.security.authc.realms.oidc.*.ssl.key,reindex.ssl.key_passphrase,xpack.security.authc.realms.ldap.*.ssl.cipher_suites,xpack.security.authc.realms.ldap.*.ssl.keystore.algorithm,xpack.http.ssl.keystore.algorithm,xpack.security.authc.realms.saml.*.ssl.keystore.password,xpack.http.ssl.supported_protocols,xpack.http.ssl.keystore.path,xpack.security.authc.realms.saml.*.encryption.keystore.algorithm,xpack.notification.email.ssl.certificate,xpack.security.http.ssl.truststore.path,xpack.http.ssl.truststore.password,xpack.security.transport.ssl.verification_mode,xpack.security.authc.realms.active_directory.*.ssl.truststore.password,xpack.security.authc.realms.saml.*.ssl.truststore.type,xpack.security.authc.realms.oidc.*.ssl.certificate_authorities,xpack.security.authc.realms.ldap.*.ssl.keystore.password,xpack.security.transport.ssl.certificate,xpack.monitoring.exporters.*.auth.*,xpack.security.authc.realms.oidc.*.ssl.keystore.algorithm,xpack.notification.email.ssl.key,xpack.http.ssl.keystore.key_password,xpack.security.authc.realms.saml.*.ssl.keystore.key_password,xpack.security.authc.realms.oidc.*.ssl.key_passphrase,reindex.ssl.truststore.path,xpack.notification.email.ssl.keystore.algorithm,xpack.notification.email.ssl.verification_mode,transport.profiles.*.xpack.security.ssl.truststore.password,transport.profiles.*.xpack.security.ssl.certificate,transport.profiles.*.xpack.security.*,xpack.security.authc.realms.ldap.*.ssl.truststore.password,xpack.security.authc.realms.pki.*.certificate_authorities,xpack.security.authc.realms.active_directory.*.bind_dn,xpack.http.ssl.truststore.path,xpack.http.ssl.key_passphrase,xpack.security.authc.realms.saml.*.ssl.certificate_authorities,reindex.ssl.verification_mode,xpack.http.ssl.keystore.password,xpack.notification.email.ssl.trust_restrictions.path,xpack.notification.email.ssl.keystore.key_password,xpack.security.transport.ssl.truststore.path,xpack.security.transport.ssl.trust_restrictions.path,xpack.security.http.ssl.truststore.type,xpack.http.ssl.certificate_authorities,xpack.security.authc.realms.active_directory.*.bind_password,xpack.security.transport.ssl.keystore.type,xpack.security.authc.realms.active_directory.*.ssl.keystore.path,xpack.security.http.ssl.keystore.algorithm,reindex.ssl.keystore.path,xpack.security.authc.realms.ldap.*.ssl.trust_restrictions,xpack.security.authc.realms.saml.*.ssl.verification_mode,xpack.security.authc.realms.oidc.*.ssl.cipher_suites,xpack.security.authc.realms.ldap.*.ssl.supported_protocols,transport.profiles.*.xpack.security.ssl.key,xpack.security.transport.ssl.keystore.password,reindex.ssl.cipher_suites,xpack.security.authc.realms.pki.*.truststore.algorithm,xpack.monitoring.exporters.*.auth.password,xpack.security.transport.ssl.keystore.algorithm,transport.profiles.*.xpack.security.ssl.keystore.type,xpack.security.authc.realms.saml.*.encryption.keystore.path,xpack.notification.email.ssl.client_authentication,transport.profiles.*.xpack.security.ssl.client_authentication,xpack.security.authc.realms.ldap.*.ssl.key_passphrase,reindex.ssl.keystore.algorithm,xpack.security.authc.realms.oidc.*.ssl.supported_protocols,xpack.http.ssl.cipher_suites,xpack.security.authc.realms.active_directory.*.ssl.certificate_authorities,xpack.security.authc.realms.saml.*.ssl.client_authentication,xpack.security.authc.realms.oidc.*.ssl.client_authentication,xpack.monitoring.exporters.*.auth.username,xpack.security.authc.realms.active_directory.*.ssl.truststore.type,xpack.security.http.ssl.client_authentication,xpack.security.authc.accept_default_password,xpack.security.authc.realms.ldap.*.ssl.keystore.path,xpack.notification.email.ssl.keystore.path,xpack.security.authc.realms.active_directory.*.ssl.keystore.algorithm,xpack.security.transport.ssl.key,xpack.security.transport.ssl.truststore.algorithm,xpack.security.authc.realms.ldap.*.ssl.truststore.type,xpack.http.ssl.certificate,transport.profiles.*.xpack.security.ssl.certificate_authorities,xpack.security.authc.realms.active_directory.*.ssl.cipher_suites,transport.profiles.*.xpack.security.ssl.keystore.password,xpack.notification.email.ssl.truststore.algorithm,xpack.security.authc.realms.saml.*.ssl.key,xpack.notification.email.ssl.cipher_suites,xpack.security.authc.realms.saml.*.ssl.keystore.type,xpack.security.authc.realms.saml.*.ssl.keystore.algorithm,xpack.security.authc.realms.oidc.*.ssl.trust_restrictions,xpack.security.authc.realms.saml.*.signing.keystore.type,xpack.security.authc.realms.active_directory.*.ssl.certificate,reindex.ssl.truststore.type,xpack.security.transport.ssl.cipher_suites,xpack.security.http.ssl.certificate,xpack.http.ssl.keystore.type,xpack.security.http.ssl.certificate_authorities,xpack.security.authc.realms.oidc.*.ssl.truststore.password,transport.profiles.*.xpack.security.ssl.truststore.type,transport.profiles.*.xpack.security.ssl.cipher_suites,xpack.security.authc.realms.saml.*.ssl.cipher_suites,xpack.security.authc.realms.saml.*.ssl.key_passphrase,xpack.security.authc.realms.active_directory.*.ssl.truststore.algorithm,xpack.security.authc.realms.saml.*.signing.keystore.algorithm,xpack.security.authc.realms.ldap.*.bind_dn,xpack.security.authc.realms.saml.*.encryption.certificate,xpack.security.authc.realms.oidc.*.ssl.keystore.key_password,xpack.http.ssl.key,xpack.notification.email.ssl.truststore.type,xpack.security.authc.realms.saml.*.ssl.truststore.path,xpack.security.authc.realms.active_directory.*.ssl.key,xpack.security.authc.realms.saml.*.ssl.truststore.password,xpack.security.authc.realms.pki.*.truststore.password,reindex.ssl.keystore.password,xpack.security.authc.realms.oidc.*.ssl.certificate,xpack.http.ssl.verification_mode,reindex.ssl.certificate_authorities,transport.profiles.*.xpack.security.ssl.verification_mode,xpack.security.http.ssl.keystore.key_password,transport.profiles.*.xpack.security.ssl.keystore.key_password,xpack.notification.email.ssl.certificate_authorities,xpack.notification.email.ssl.keystore.type,xpack.monitoring.exporters.*.ssl,xpack.security.authc.realms.saml.*.ssl.trust_restrictions,xpack.security.authc.realms.ldap.*.bind_password,xpack.security.http.ssl.cipher_suites,reindex.ssl.certificate,transport.profiles.*.xpack.security.ssl.truststore.algorithm,xpack.security.authc.realms.active_directory.*.ssl.keystore.type,xpack.security.transport.ssl.truststore.type,xpack.security.transport.ssl.certificate_authorities,xpack.security.authc.realms.active_directory.*.ssl.trust_restrictions,xpack.security.authc.realms.saml.*.ssl.supported_protocols,xpack.security.transport.ssl.keystore.path,xpack.notification.email.ssl.truststore.path,xpack.http.ssl.trust_restrictions.path,xpack.security.http.ssl.key_passphrase,xpack.security.http.ssl.trust_restrictions.path,xpack.http.ssl.truststore.type,transport.profiles.*.xpack.security.ssl.keystore.algorithm,xpack.security.transport.ssl.client_authentication,xpack.security.authc.realms.ldap.*.ssl.client_authentication,reindex.ssl.client_authentication,xpack.security.authc.realms.oidc.*.ssl.truststore.path,reindex.ssl.keystore.type,xpack.http.ssl.client_authentication,xpack.security.transport.ssl.key_passphrase,xpack.monitoring.exporters.*.ssl.*,xpack.security.hide_settings,xpack.security.authc.realms.saml.*.signing.certificate,xpack.security.authc.realms.saml.*.encryption.keystore.type,xpack.security.authc.realms.ldap.*.hostname_verification,xpack.security.authc.realms.active_directory.*.ssl.truststore.path,xpack.notification.email.ssl.keystore.password,reindex.ssl.key,xpack.security.authc.realms.active_directory.*.ssl.key_passphrase,transport.profiles.*.xpack.security.ssl.keystore.path,xpack.security.http.ssl.keystore.path,xpack.security.authc.realms.ldap.*.ssl.truststore.algorithm,xpack.security.authc.realms.ldap.*.ssl.keystore.type,xpack.security.authc.realms.active_directory.*.ssl.client_authentication,xpack.security.authc.realms.oidc.*.ssl.keystore.type,xpack.security.authc.realms.saml.*.ssl.certificate,xpack.notification.email.ssl.key_passphrase,xpack.security.authc.realms.active_directory.*.ssl.keystore.password,xpack.security.authc.realms.oidc.*.ssl.truststore.algorithm,transport.profiles.*.xpack.security.ssl.key_passphrase,xpack.security.authc.realms.ldap.*.ssl.certificate}
org.elasticsearch.discovery.MasterNotDiscoveredException: null
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:218) [elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:252) [elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:598) [elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) [elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T14:29:47,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] no known master node, scheduling a retry
[2021-05-03T14:29:47,631][DEBUG][o.e.a.s.m.TransportMasterNodeAction] [master1] timed out while retrying [cluster:monitor/state] after failure (timeout [30s])
[ deleted repeated messages ]
[2021-05-03T14:30:16,586][INFO ][o.e.c.s.MasterService ] [master1] node-left[{master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} reason: disconnected], term: 41, version: 2131741, delta: removed {{master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}
[2021-05-03T14:30:16,969][INFO ][o.e.c.s.ClusterApplierService] [master1] removed {{master0}{xexrc0uXQUWgwFDWEpAouA}{VYK7ojdqQU6DH676OtMWyA}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}, term: 41, version: 2131741, reason: Publication{term=41, version=2131741}
[2021-05-03T14:32:29,377][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5601891] overhead, spent [250ms] collecting in the last [1s]
[2021-05-03T14:33:19,586][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5601941] overhead, spent [277ms] collecting in the last [1s]
[2021-05-03T14:33:59,651][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5601981] overhead, spent [307ms] collecting in the last [1s]
[2021-05-03T14:34:09,696][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5601991] overhead, spent [265ms] collecting in the last [1s]
[2021-05-03T14:34:19,732][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602001] overhead, spent [316ms] collecting in the last [1s]
[2021-05-03T14:35:19,957][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602061] overhead, spent [267ms] collecting in the last [1s]
[2021-05-03T14:35:39,984][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602081] overhead, spent [261ms] collecting in the last [1s]
[2021-05-03T14:37:00,086][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602161] overhead, spent [252ms] collecting in the last [1s]
[2021-05-03T14:37:49,250][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602210] overhead, spent [310ms] collecting in the last [1.1s]
[2021-05-03T14:38:09,367][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602230] overhead, spent [282ms] collecting in the last [1s]
[2021-05-03T14:38:59,403][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602280] overhead, spent [251ms] collecting in the last [1s]
[2021-05-03T14:39:29,413][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602310] overhead, spent [257ms] collecting in the last [1s]
[2021-05-03T14:40:19,627][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602360] overhead, spent [268ms] collecting in the last [1s]
[2021-05-03T14:40:49,774][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602390] overhead, spent [260ms] collecting in the last [1s]
[2021-05-03T14:41:19,863][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602420] overhead, spent [268ms] collecting in the last [1s]
[2021-05-03T14:42:09,994][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602470] overhead, spent [312ms] collecting in the last [1s]
[2021-05-03T14:42:30,000][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602490] overhead, spent [269ms] collecting in the last [1s]
[2021-05-03T14:43:09,148][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602529] overhead, spent [323ms] collecting in the last [1s]
[2021-05-03T14:46:20,273][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602720] overhead, spent [258ms] collecting in the last [1s]
[2021-05-03T14:46:49,307][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602749] overhead, spent [321ms] collecting in the last [1s]
[2021-05-03T14:47:29,376][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602789] overhead, spent [257ms] collecting in the last [1s]
[2021-05-03T14:47:39,436][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602799] overhead, spent [282ms] collecting in the last [1s]
[2021-05-03T14:47:49,503][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602809] overhead, spent [278ms] collecting in the last [1s]
[2021-05-03T14:48:18,102][INFO ][o.e.c.s.MasterService ] [master1] node-join[{master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} join existing leader], term: 41, version: 2131743, delta: added {{master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}
[2021-05-03T14:48:28,321][INFO ][o.e.c.c.C.CoordinatorPublication] [master1] after [10s] publication of cluster state version [2131743] is still waiting for {master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} [SENT_PUBLISH_REQUEST]
[2021-05-03T14:48:29,599][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602849] overhead, spent [272ms] collecting in the last [1s]
[2021-05-03T14:48:48,323][INFO ][o.e.c.s.ClusterApplierService] [master1] added {{master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}, term: 41, version: 2131743, reason: Publication{term=41, version=2131743}
[2021-05-03T14:48:48,356][WARN ][o.e.c.c.C.CoordinatorPublication] [master1] after [30s] publication of cluster state version [2131743] is still waiting for {master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} [SENT_PUBLISH_REQUEST]
[2021-05-03T14:48:59,806][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602879] overhead, spent [261ms] collecting in the last [1s]
[2021-05-03T14:49:09,814][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602889] overhead, spent [289ms] collecting in the last [1s]
[2021-05-03T14:49:29,872][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5602909] overhead, spent [266ms] collecting in the last [1s]
[2021-05-03T14:55:19,666][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603258] overhead, spent [263ms] collecting in the last [1s]
[2021-05-03T14:55:29,697][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603268] overhead, spent [277ms] collecting in the last [1s]
[2021-05-03T14:56:19,759][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603318] overhead, spent [278ms] collecting in the last [1s]
[2021-05-03T14:56:39,847][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603338] overhead, spent [300ms] collecting in the last [1s]
[2021-05-03T14:57:09,023][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603367] overhead, spent [281ms] collecting in the last [1.1s]
[2021-05-03T14:57:40,064][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603398] overhead, spent [260ms] collecting in the last [1s]
[2021-05-03T14:58:00,076][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603418] overhead, spent [258ms] collecting in the last [1s]
[2021-05-03T15:00:09,598][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603547] overhead, spent [257ms] collecting in the last [1s]
[2021-05-03T15:00:39,609][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603577] overhead, spent [266ms] collecting in the last [1s]
[2021-05-03T15:00:59,629][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603597] overhead, spent [276ms] collecting in the last [1s]
[2021-05-03T15:01:09,645][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603607] overhead, spent [258ms] collecting in the last [1s]
[2021-05-03T15:04:49,197][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5603826] overhead, spent [264ms] collecting in the last [1s]
[2021-05-03T15:07:49,387][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604006] overhead, spent [308ms] collecting in the last [1s]
[2021-05-03T15:10:29,608][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604166] overhead, spent [266ms] collecting in the last [1s]
[2021-05-03T15:10:49,676][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604186] overhead, spent [334ms] collecting in the last [1s]
[2021-05-03T15:10:59,681][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604196] overhead, spent [259ms] collecting in the last [1s]
[2021-05-03T15:11:59,771][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604256] overhead, spent [255ms] collecting in the last [1s]
[2021-05-03T15:12:19,836][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604276] overhead, spent [266ms] collecting in the last [1s]
[2021-05-03T15:12:29,919][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604286] overhead, spent [256ms] collecting in the last [1s]
[2021-05-03T15:12:39,922][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604296] overhead, spent [250ms] collecting in the last [1s]
[2021-05-03T15:15:29,441][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604465] overhead, spent [280ms] collecting in the last [1s]
[2021-05-03T15:15:49,477][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604485] overhead, spent [283ms] collecting in the last [1s]
[2021-05-03T15:17:09,661][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604565] overhead, spent [281ms] collecting in the last [1s]
[2021-05-03T15:17:19,681][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604575] overhead, spent [272ms] collecting in the last [1s]
[2021-05-03T15:18:29,847][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604645] overhead, spent [290ms] collecting in the last [1s]
[2021-05-03T15:18:59,985][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604675] overhead, spent [294ms] collecting in the last [1s]
[2021-05-03T15:19:20,066][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604695] overhead, spent [254ms] collecting in the last [1s]
[2021-05-03T15:21:49,510][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604844] overhead, spent [369ms] collecting in the last [1s]
[2021-05-03T15:22:59,603][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604914] overhead, spent [305ms] collecting in the last [1s]
[2021-05-03T15:23:30,614][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604945] overhead, spent [265ms] collecting in the last [1s]
[2021-05-03T15:23:39,617][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604954] overhead, spent [303ms] collecting in the last [1s]
[2021-05-03T15:23:59,681][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5604974] overhead, spent [258ms] collecting in the last [1s]
[2021-05-03T15:24:39,760][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605014] overhead, spent [287ms] collecting in the last [1s]
[2021-05-03T15:25:09,853][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605044] overhead, spent [303ms] collecting in the last [1s]
[2021-05-03T15:25:28,889][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605063] overhead, spent [273ms] collecting in the last [1s]
[2021-05-03T15:25:59,051][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605093] overhead, spent [286ms] collecting in the last [1s]
[2021-05-03T15:26:10,092][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605104] overhead, spent [336ms] collecting in the last [1s]
[2021-05-03T15:26:49,311][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605143] overhead, spent [288ms] collecting in the last [1s]
[2021-05-03T15:26:50,311][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605144] overhead, spent [251ms] collecting in the last [1s]
[2021-05-03T15:27:09,316][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605163] overhead, spent [262ms] collecting in the last [1s]
[2021-05-03T15:32:19,583][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605473] overhead, spent [300ms] collecting in the last [1s]
[2021-05-03T15:32:39,588][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605493] overhead, spent [267ms] collecting in the last [1s]
[2021-05-03T15:32:59,616][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605513] overhead, spent [254ms] collecting in the last [1s]
[2021-05-03T15:33:29,653][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605543] overhead, spent [251ms] collecting in the last [1s]
[2021-05-03T15:33:59,661][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605573] overhead, spent [274ms] collecting in the last [1s]
[2021-05-03T15:34:19,704][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605593] overhead, spent [291ms] collecting in the last [1s]
[2021-05-03T15:35:19,995][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605653] overhead, spent [325ms] collecting in the last [1s]
[2021-05-03T15:36:50,311][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605743] overhead, spent [253ms] collecting in the last [1s]
[2021-05-03T15:37:39,480][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605792] overhead, spent [288ms] collecting in the last [1s]
[2021-05-03T15:37:49,493][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605802] overhead, spent [280ms] collecting in the last [1s]
[2021-05-03T15:38:09,515][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605822] overhead, spent [320ms] collecting in the last [1s]
[2021-05-03T15:39:19,675][INFO ][o.e.m.j.JvmGcMonitorService] [master1] [gc][5605892] overhead, spent [299ms] collecting in the last [1s]
[2021-05-03T15:39:23,520][INFO ][o.e.n.Node ] [master1] stopping ...
[2021-05-03T15:39:23,556][INFO ][o.e.n.Node ] [master1] stopped
[2021-05-03T15:39:23,556][INFO ][o.e.n.Node ] [master1] closing ...
[2021-05-03T15:39:23,588][INFO ][o.e.n.Node ] [master1] closed
[2021-05-03T15:43:11,028][INFO ][o.e.e.NodeEnvironment ] [master1] using [1] data paths, mounts [[/ (/dev/sda1)]], net usable_space [29.3gb], net total_space [41.2gb], types [ext4]
[2021-05-03T15:43:11,031][INFO ][o.e.e.NodeEnvironment ] [master1] heap size [11.9gb], compressed ordinary object pointers [true]
[2021-05-03T15:43:33,854][INFO ][o.e.n.Node ] [master1] node name [master1], node ID [Ig33yWUwTv-UNSwIhhfNWg], cluster name [elasticsearch]
[2021-05-03T15:43:33,855][INFO ][o.e.n.Node ] [master1] version[7.5.2], pid[29122], build[default/deb/8bec50e1e0ad29dad5653712cf3bb580cd1afcdf/2020-01-15T12:11:52.313576Z], OS[Linux/4.4.0-203-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/13.0.1/13.0.1+9]
[2021-05-03T15:43:33,855][INFO ][o.e.n.Node ] [master1] JVM home [/usr/share/elasticsearch/jdk]
[2021-05-03T15:43:33,856][INFO ][o.e.n.Node ] [master1] JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=COMPAT, -Xms12g, -Xmx12g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+DisableExplicitGC, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Djdk.io.permissionsUseCanonicalPath=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -XX:MaxDirectMemorySize=6442450944, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/etc/elasticsearch, -Des.distribution.flavor=default, -Des.distribution.type=deb, -Des.bundled_jdk=true]
[2021-05-03T15:43:37,511][INFO ][o.e.p.PluginsService ] [master1] loaded module [aggs-matrix-stats]
[2021-05-03T15:43:37,512][INFO ][o.e.p.PluginsService ] [master1] loaded module [analysis-common]
[2021-05-03T15:43:37,512][INFO ][o.e.p.PluginsService ] [master1] loaded module [flattened]
[2021-05-03T15:43:37,512][INFO ][o.e.p.PluginsService ] [master1] loaded module [frozen-indices]
[2021-05-03T15:43:37,512][INFO ][o.e.p.PluginsService ] [master1] loaded module [ingest-common]
[2021-05-03T15:43:37,512][INFO ][o.e.p.PluginsService ] [master1] loaded module [ingest-geoip]
[2021-05-03T15:43:37,513][INFO ][o.e.p.PluginsService ] [master1] loaded module [ingest-user-agent]
[2021-05-03T15:43:37,513][INFO ][o.e.p.PluginsService ] [master1] loaded module [lang-expression]
[2021-05-03T15:43:37,513][INFO ][o.e.p.PluginsService ] [master1] loaded module [lang-mustache]
[2021-05-03T15:43:37,513][INFO ][o.e.p.PluginsService ] [master1] loaded module [lang-painless]
[2021-05-03T15:43:37,513][INFO ][o.e.p.PluginsService ] [master1] loaded module [mapper-extras]
[2021-05-03T15:43:37,513][INFO ][o.e.p.PluginsService ] [master1] loaded module [parent-join]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [percolator]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [rank-eval]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [reindex]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [repository-url]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [search-business-rules]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [spatial]
[2021-05-03T15:43:37,514][INFO ][o.e.p.PluginsService ] [master1] loaded module [systemd]
[2021-05-03T15:43:37,515][INFO ][o.e.p.PluginsService ] [master1] loaded module [transform]
[2021-05-03T15:43:37,515][INFO ][o.e.p.PluginsService ] [master1] loaded module [transport-netty4]
[2021-05-03T15:43:37,515][INFO ][o.e.p.PluginsService ] [master1] loaded module [vectors]
[2021-05-03T15:43:37,515][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-analytics]
[2021-05-03T15:43:37,515][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-ccr]
[2021-05-03T15:43:37,515][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-core]
[2021-05-03T15:43:37,516][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-deprecation]
[2021-05-03T15:43:37,516][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-enrich]
[2021-05-03T15:43:37,516][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-graph]
[2021-05-03T15:43:37,516][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-ilm]
[2021-05-03T15:43:37,516][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-logstash]
[2021-05-03T15:43:37,516][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-ml]
[2021-05-03T15:43:37,517][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-monitoring]
[2021-05-03T15:43:37,517][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-rollup]
[2021-05-03T15:43:37,517][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-security]
[2021-05-03T15:43:37,517][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-sql]
[2021-05-03T15:43:37,517][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-voting-only-node]
[2021-05-03T15:43:37,517][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-watcher]
[2021-05-03T15:43:37,518][INFO ][o.e.p.PluginsService ] [master1] no plugins loaded
[2021-05-03T15:43:41,042][INFO ][o.e.d.DiscoveryModule ] [master1] using discovery type [zen] and seed hosts providers [settings]
[2021-05-03T15:43:41,669][INFO ][o.e.n.Node ] [master1] initialized
[2021-05-03T15:43:41,669][INFO ][o.e.n.Node ] [master1] starting ...
[2021-05-03T15:43:41,809][INFO ][o.e.t.TransportService ] [master1] publish_address {192.168.0.11:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}, {192.168.0.11:9300}
[2021-05-03T15:43:46,691][INFO ][o.e.b.BootstrapChecks ] [master1] bound or publishing to a non-loopback address, enforcing bootstrap checks
[2021-05-03T15:43:46,693][INFO ][o.e.c.c.Coordinator ] [master1] cluster UUID [9ebN7hmbR3O_-EqkjWDXwg]
[2021-05-03T15:43:56,702][WARN ][o.e.c.c.ClusterFormationFailureHelper] [master1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [RBMXwoyITeqpwJlY7VFFhA, xexrc0uXQUWgwFDWEpAouA, Ig33yWUwTv-UNSwIhhfNWg], have discovered [{master1}{Ig33yWUwTv-UNSwIhhfNWg}{iUGHa9kFTjK57EFpCcNWbA}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, {master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}, {master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}] which is a quorum; discovery will continue using [192.168.0.23:9300, 192.168.0.24:9300, 192.168.0.25:9300, 192.168.0.10:9300, 127.0.1.1:9300, 192.168.0.12:9300] from hosts providers and [{master1}{Ig33yWUwTv-UNSwIhhfNWg}{iUGHa9kFTjK57EFpCcNWbA}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}] from last-known cluster state; node term 47, last-accepted version 2131746 in term 41
[2021-05-03T15:43:59,345][INFO ][o.e.c.c.JoinHelper ] [master1] failed to join {master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} with JoinRequest{sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{iUGHa9kFTjK57EFpCcNWbA}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, optionalJoin=Optional[Join{term=48, lastAcceptedTerm=41, lastAcceptedVersion=2131746, sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{iUGHa9kFTjK57EFpCcNWbA}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, targetNode={master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}]}
org.elasticsearch.transport.RemoteTransportException: [master0][192.168.0.10:9300][internal:cluster/coordination/join]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: incoming term 48 does not match current term 49
at org.elasticsearch.cluster.coordination.CoordinationState.handleJoin(CoordinationState.java:223) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator.handleJoin(Coordinator.java:979) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.Optional.ifPresent(Optional.java:176) ~[?:?]
at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:524) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:490) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.ConnectionManager.connectToNode(ConnectionManager.java:121) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:370) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:354) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:477) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:125) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T15:43:59,571][INFO ][o.e.c.c.JoinHelper ] [master1] failed to join {master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true} with JoinRequest{sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{iUGHa9kFTjK57EFpCcNWbA}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, optionalJoin=Optional[Join{term=47, lastAcceptedTerm=41, lastAcceptedVersion=2131746, sourceNode={master1}{Ig33yWUwTv-UNSwIhhfNWg}{iUGHa9kFTjK57EFpCcNWbA}{192.168.0.11}{192.168.0.11:9300}{im}{xpack.installed=true}, targetNode={master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true}}]}
org.elasticsearch.transport.RemoteTransportException: [master0][192.168.0.10:9300][internal:cluster/coordination/join]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: incoming term 47 does not match current term 49
at org.elasticsearch.cluster.coordination.CoordinationState.handleJoin(CoordinationState.java:223) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator.handleJoin(Coordinator.java:979) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.Optional.ifPresent(Optional.java:176) ~[?:?]
at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:524) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator.access$000(Coordinator.java:97) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$2.onResponse(Coordinator.java:503) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.cluster.coordination.Coordinator$2.onResponse(Coordinator.java:499) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1110) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:221) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) ~[elasticsearch-7.5.2.jar:7.5.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.5.2.jar:7.5.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:830) [?:?]
[2021-05-03T15:44:10,799][WARN ][o.e.g.IncrementalClusterStateWriter] [master1] writing cluster state took [11004ms] which is above the warn threshold of [10s]; wrote metadata for [4361] indices and skipped [0] unchanged indices
[2021-05-03T15:44:10,807][INFO ][o.e.c.s.ClusterApplierService] [master1] master node changed {previous [], current [{master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}]}, added {{data09}{dN08oWhmSNObQgeb9xH_-A}{9aLNT59cQZutpH2xd1LzTQ}{192.168.0.29}{192.168.0.29:9300}{di}{xpack.installed=true},{data06}{NqAsywRoTVSuqTxD8cmq3A}{ffiDh9EhT0OpZ0Suj4hHcQ}{192.168.0.26}{192.168.0.26:9300}{di}{xpack.installed=true},{master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true},{data04}{M7yBnQeXTwCohL8IWN4PBg}{HRi5FXR0S_m2YlWM5zAMCw}{192.168.0.24}{192.168.0.24:9300}{di}{xpack.installed=true},{master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true},{data12b}{Fs5fJqYNSLOPK7OVW7qaYQ}{2SeThcYbQ5uUG6c7JyP_Qg}{192.168.0.32}{192.168.0.32:9301}{di}{xpack.installed=true},{data10b}{WBnLOtGgS6KXAV8FsiFYjg}{D1xTZOswTAiMc3h1YMae3Q}{192.168.0.30}{192.168.0.30:9301}{di}{xpack.installed=true},{data11b}{6FVMYvwiQD-W6fpHtVXPfg}{dAQSQFZMSjKq6kxq8cc98g}{192.168.0.31}{192.168.0.31:9301}{di}{xpack.installed=true},{data17a}{x-YWJyTxRoCh8Lf4oS0qWQ}{MqzCX5bVRCa7MWZ3e7CtZg}{192.168.0.37}{192.168.0.37:9300}{di}{xpack.installed=true},{data17b}{7uaz7rwQRg-UQQ4WLhclBg}{SXb-eNTSTP-9-dO7pgnEfQ}{192.168.0.37}{192.168.0.37:9301}{di}{xpack.installed=true},{data11a}{EI9MEUGwQP6yuwlmVsH-EA}{QRz_oL35TyOvnBqTWqWgEA}{192.168.0.31}{192.168.0.31:9300}{di}{xpack.installed=true},{data16b}{2afokmtOQGGzfFLz3v4_Bg}{Lo4izLlTQqSMfbrf21aAkw}{192.168.0.36}{192.168.0.36:9301}{di}{xpack.installed=true},{data03}{9MyOkgP3TuSevNQJkEmOnw}{MBDXNqQOR6G05JiS0UQlEw}{192.168.0.23}{192.168.0.23:9300}{di}{xpack.installed=true},{data12a}{22tiK4-STyC2XRJ01nWQ3A}{rKiBCqOFTGKTBesQCzI7aw}{192.168.0.32}{192.168.0.32:9300}{di}{xpack.installed=true},{data15a}{0sFGQdJwSNermm8lSE7v6w}{TU-BKgdwQO2xOnhMx6o-ow}{192.168.0.35}{192.168.0.35:9300}{di}{xpack.installed=true},{data13a}{vF-2rZ1ZTuG8rcYDTScDbQ}{NY6x69ejT9ehohT-uXKyww}{192.168.0.33}{192.168.0.33:9300}{di}{xpack.installed=true},{data07}{uYjW78V_QAaAQPHjUSNjjQ}{ZA9njwKuRgOEdIllyccwqQ}{192.168.0.27}{192.168.0.27:9300}{di}{xpack.installed=true},{data18b}{5HTiO0Z8RM2tRi0kaNQQ8w}{O-T8IWgXQIekz9QHQyKy4g}{192.168.0.38}{192.168.0.38:9301}{di}{xpack.installed=true},{data05}{3sy0vv41TXGy4Yt8ywpXVw}{AoblOEIlSF2ZWQSqEERqwA}{192.168.0.25}{192.168.0.25:9300}{di}{xpack.installed=true},{data16a}{SzyPNxe6Rm6H9ucP26H6Mw}{PALqrfCdR6CAvYfrioYAZQ}{192.168.0.36}{192.168.0.36:9300}{di}{xpack.installed=true},{data13b}{mAQ5jgAMQjaYzwpVE83kcg}{LBsGSuLJR5Kg8iWCA-SX1A}{192.168.0.33}{192.168.0.33:9301}{di}{xpack.installed=true},{data08}{GIaGadJ0RhyBUePNzWpBdA}{fz0VawBhT_6hIre0uymIfw}{192.168.0.28}{192.168.0.28:9300}{di}{xpack.installed=true},{data18a}{_MeUtbWCQK6WAcOWs6L4fQ}{qjYEjHO0Qp-pysLXoM8iYA}{192.168.0.38}{192.168.0.38:9300}{di}{xpack.installed=true},{data15b}{bT-KQ6TgQHWYl3pLMjj9MQ}{I9wc6TOxQ2WkqbOZ89g8XQ}{192.168.0.35}{192.168.0.35:9301}{di}{xpack.installed=true},{data10a}{Cl-VWQCjQ3OStxiPmdIXrQ}{KjVwOpXuQCKLy2GQFBluZg}{192.168.0.30}{192.168.0.30:9300}{di}{xpack.installed=true}}, term: 49, version: 2131752, reason: ApplyCommitRequest{term=49, version=2131752, sourceNode={master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}}
[2021-05-03T15:44:10,931][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.cluster_concurrent_rebalance] from [2] to [20]
[2021-05-03T15:44:10,932][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.same_shard.host] from [false] to [true]
[2021-05-03T15:44:10,932][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.low] from [85%] to [150gb]
[2021-05-03T15:44:10,932][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.high] from [90%] to [100gb]
[2021-05-03T15:44:10,932][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.flood_stage] from [95%] to [50gb]
[2021-05-03T15:44:10,932][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.node_initial_primaries_recoveries] from [4] to [16]
[2021-05-03T15:44:10,933][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.node_concurrent_incoming_recoveries] from [2] to [20]
[2021-05-03T15:44:10,933][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.node_concurrent_outgoing_recoveries] from [2] to [20]
[2021-05-03T15:44:10,933][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [xpack.monitoring.elasticsearch.collection.enabled] from [true] to [false]
[2021-05-03T15:44:10,933][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.low] from [85%] to [150gb]
[2021-05-03T15:44:10,934][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.high] from [90%] to [100gb]
[2021-05-03T15:44:10,934][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.flood_stage] from [95%] to [50gb]
[2021-05-03T15:44:10,934][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [search.max_buckets] from [10000] to [20000]
[2021-05-03T15:44:10,934][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [indices.recovery.max_bytes_per_sec] from [40mb] to [120mb]
[2021-05-03T15:44:11,197][INFO ][o.e.l.LicenseService ] [master1] license [c01f4c23-89af-4849-841d-675391762534] mode [basic] - valid
[2021-05-03T15:44:11,229][INFO ][o.e.h.AbstractHttpServerTransport] [master1] publish_address {192.168.0.11:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}, {192.168.0.11:9200}
[2021-05-03T15:44:11,229][INFO ][o.e.n.Node ] [master1] started
[2021-05-03T15:44:27,426][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Received ban for the parent [dN08oWhmSNObQgeb9xH_-A:214934874] on the node [Ig33yWUwTv-UNSwIhhfNWg], reason: [by user request]
[2021-05-03T15:44:27,427][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Received ban for the parent [22tiK4-STyC2XRJ01nWQ3A:3325538968] on the node [Ig33yWUwTv-UNSwIhhfNWg], reason: [by user request]
[2021-05-03T15:44:35,429][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Removing ban for the parent [22tiK4-STyC2XRJ01nWQ3A:3325538968] on the node [Ig33yWUwTv-UNSwIhhfNWg]
[2021-05-03T15:44:35,673][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Removing ban for the parent [dN08oWhmSNObQgeb9xH_-A:214934874] on the node [Ig33yWUwTv-UNSwIhhfNWg]
[2021-05-03T15:48:24,999][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Received ban for the parent [Cl-VWQCjQ3OStxiPmdIXrQ:165291719] on the node [Ig33yWUwTv-UNSwIhhfNWg], reason: [by user request]
[2021-05-03T15:48:24,999][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Received ban for the parent [3sy0vv41TXGy4Yt8ywpXVw:218731516] on the node [Ig33yWUwTv-UNSwIhhfNWg], reason: [by user request]
[2021-05-03T15:48:30,405][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Removing ban for the parent [3sy0vv41TXGy4Yt8ywpXVw:218731516] on the node [Ig33yWUwTv-UNSwIhhfNWg]
[2021-05-03T15:48:36,034][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Removing ban for the parent [Cl-VWQCjQ3OStxiPmdIXrQ:165291719] on the node [Ig33yWUwTv-UNSwIhhfNWg]
[2021-05-03T15:49:39,779][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Received ban for the parent [3sy0vv41TXGy4Yt8ywpXVw:218820445] on the node [Ig33yWUwTv-UNSwIhhfNWg], reason: [by user request]
[2021-05-03T15:49:45,510][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Removing ban for the parent [3sy0vv41TXGy4Yt8ywpXVw:218820445] on the node [Ig33yWUwTv-UNSwIhhfNWg]
[2021-05-03T15:49:49,368][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Received ban for the parent [22tiK4-STyC2XRJ01nWQ3A:3325759373] on the node [Ig33yWUwTv-UNSwIhhfNWg], reason: [by user request]
[2021-05-03T15:49:55,487][DEBUG][o.e.a.a.c.n.t.c.TransportCancelTasksAction] [master1] Removing ban for the parent [22tiK4-STyC2XRJ01nWQ3A:3325759373] on the node [Ig33yWUwTv-UNSwIhhfNWg]
[2021-05-03T15:52:53,865][INFO ][o.e.n.Node ] [master1] stopping ...
[2021-05-03T15:52:53,881][INFO ][o.e.c.c.Coordinator ] [master1] master node [{master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}] failed, restarting discovery
org.elasticsearch.transport.NodeDisconnectedException: [master2][192.168.0.12:9300][disconnected] disconnected
[2021-05-03T15:52:53,913][INFO ][o.e.n.Node ] [master1] stopped
[2021-05-03T15:52:53,914][INFO ][o.e.n.Node ] [master1] closing ...
[2021-05-03T15:52:53,925][INFO ][o.e.n.Node ] [master1] closed
[2021-05-03T16:08:45,947][INFO ][o.e.e.NodeEnvironment ] [master1] using [1] data paths, mounts [[/ (/dev/sda1)]], net usable_space [28.8gb], net total_space [41.2gb], types [ext4]
[2021-05-03T16:08:45,951][INFO ][o.e.e.NodeEnvironment ] [master1] heap size [11.9gb], compressed ordinary object pointers [true]
[2021-05-03T16:08:55,378][INFO ][o.e.n.Node ] [master1] node name [master1], node ID [Ig33yWUwTv-UNSwIhhfNWg], cluster name [elasticsearch]
[2021-05-03T16:08:55,379][INFO ][o.e.n.Node ] [master1] version[7.5.2], pid[2242], build[default/deb/8bec50e1e0ad29dad5653712cf3bb580cd1afcdf/2020-01-15T12:11:52.313576Z], OS[Linux/4.15.0-142-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/13.0.1/13.0.1+9]
[2021-05-03T16:08:55,379][INFO ][o.e.n.Node ] [master1] JVM home [/usr/share/elasticsearch/jdk]
[2021-05-03T16:08:55,379][INFO ][o.e.n.Node ] [master1] JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=COMPAT, -Xms12g, -Xmx12g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+DisableExplicitGC, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Djdk.io.permissionsUseCanonicalPath=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -XX:MaxDirectMemorySize=6442450944, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/etc/elasticsearch, -Des.distribution.flavor=default, -Des.distribution.type=deb, -Des.bundled_jdk=true]
[2021-05-03T16:08:57,984][INFO ][o.e.p.PluginsService ] [master1] loaded module [aggs-matrix-stats]
[2021-05-03T16:08:57,985][INFO ][o.e.p.PluginsService ] [master1] loaded module [analysis-common]
[2021-05-03T16:08:57,985][INFO ][o.e.p.PluginsService ] [master1] loaded module [flattened]
[2021-05-03T16:08:57,986][INFO ][o.e.p.PluginsService ] [master1] loaded module [frozen-indices]
[2021-05-03T16:08:57,986][INFO ][o.e.p.PluginsService ] [master1] loaded module [ingest-common]
[2021-05-03T16:08:57,986][INFO ][o.e.p.PluginsService ] [master1] loaded module [ingest-geoip]
[2021-05-03T16:08:57,987][INFO ][o.e.p.PluginsService ] [master1] loaded module [ingest-user-agent]
[2021-05-03T16:08:57,987][INFO ][o.e.p.PluginsService ] [master1] loaded module [lang-expression]
[2021-05-03T16:08:57,987][INFO ][o.e.p.PluginsService ] [master1] loaded module [lang-mustache]
[2021-05-03T16:08:57,988][INFO ][o.e.p.PluginsService ] [master1] loaded module [lang-painless]
[2021-05-03T16:08:57,989][INFO ][o.e.p.PluginsService ] [master1] loaded module [mapper-extras]
[2021-05-03T16:08:57,989][INFO ][o.e.p.PluginsService ] [master1] loaded module [parent-join]
[2021-05-03T16:08:57,989][INFO ][o.e.p.PluginsService ] [master1] loaded module [percolator]
[2021-05-03T16:08:57,989][INFO ][o.e.p.PluginsService ] [master1] loaded module [rank-eval]
[2021-05-03T16:08:57,990][INFO ][o.e.p.PluginsService ] [master1] loaded module [reindex]
[2021-05-03T16:08:57,991][INFO ][o.e.p.PluginsService ] [master1] loaded module [repository-url]
[2021-05-03T16:08:57,991][INFO ][o.e.p.PluginsService ] [master1] loaded module [search-business-rules]
[2021-05-03T16:08:57,991][INFO ][o.e.p.PluginsService ] [master1] loaded module [spatial]
[2021-05-03T16:08:57,991][INFO ][o.e.p.PluginsService ] [master1] loaded module [systemd]
[2021-05-03T16:08:57,991][INFO ][o.e.p.PluginsService ] [master1] loaded module [transform]
[2021-05-03T16:08:57,992][INFO ][o.e.p.PluginsService ] [master1] loaded module [transport-netty4]
[2021-05-03T16:08:57,992][INFO ][o.e.p.PluginsService ] [master1] loaded module [vectors]
[2021-05-03T16:08:57,992][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-analytics]
[2021-05-03T16:08:57,993][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-ccr]
[2021-05-03T16:08:57,994][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-core]
[2021-05-03T16:08:57,994][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-deprecation]
[2021-05-03T16:08:57,995][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-enrich]
[2021-05-03T16:08:57,995][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-graph]
[2021-05-03T16:08:57,995][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-ilm]
[2021-05-03T16:08:57,996][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-logstash]
[2021-05-03T16:08:57,996][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-ml]
[2021-05-03T16:08:57,996][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-monitoring]
[2021-05-03T16:08:57,996][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-rollup]
[2021-05-03T16:08:57,996][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-security]
[2021-05-03T16:08:57,997][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-sql]
[2021-05-03T16:08:57,997][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-voting-only-node]
[2021-05-03T16:08:57,997][INFO ][o.e.p.PluginsService ] [master1] loaded module [x-pack-watcher]
[2021-05-03T16:08:57,997][INFO ][o.e.p.PluginsService ] [master1] no plugins loaded
[2021-05-03T16:09:01,267][INFO ][o.e.d.DiscoveryModule ] [master1] using discovery type [zen] and seed hosts providers [settings]
[2021-05-03T16:09:01,881][INFO ][o.e.n.Node ] [master1] initialized
[2021-05-03T16:09:01,882][INFO ][o.e.n.Node ] [master1] starting ...
[2021-05-03T16:09:02,016][INFO ][o.e.t.TransportService ] [master1] publish_address {192.168.0.11:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}, {192.168.0.11:9300}
[2021-05-03T16:09:08,026][INFO ][o.e.b.BootstrapChecks ] [master1] bound or publishing to a non-loopback address, enforcing bootstrap checks
[2021-05-03T16:09:08,029][INFO ][o.e.c.c.Coordinator ] [master1] cluster UUID [9ebN7hmbR3O_-EqkjWDXwg]
[2021-05-03T16:09:09,154][INFO ][o.e.c.s.ClusterApplierService] [master1] master node changed {previous [], current [{master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}]}, added {{data10a}{Cl-VWQCjQ3OStxiPmdIXrQ}{KjVwOpXuQCKLy2GQFBluZg}{192.168.0.30}{192.168.0.30:9300}{di}{xpack.installed=true},{data05}{3sy0vv41TXGy4Yt8ywpXVw}{AoblOEIlSF2ZWQSqEERqwA}{192.168.0.25}{192.168.0.25:9300}{di}{xpack.installed=true},{data03}{9MyOkgP3TuSevNQJkEmOnw}{MBDXNqQOR6G05JiS0UQlEw}{192.168.0.23}{192.168.0.23:9300}{di}{xpack.installed=true},{data11a}{EI9MEUGwQP6yuwlmVsH-EA}{QRz_oL35TyOvnBqTWqWgEA}{192.168.0.31}{192.168.0.31:9300}{di}{xpack.installed=true},{data16a}{SzyPNxe6Rm6H9ucP26H6Mw}{PALqrfCdR6CAvYfrioYAZQ}{192.168.0.36}{192.168.0.36:9300}{di}{xpack.installed=true},{data15b}{bT-KQ6TgQHWYl3pLMjj9MQ}{I9wc6TOxQ2WkqbOZ89g8XQ}{192.168.0.35}{192.168.0.35:9301}{di}{xpack.installed=true},{data16b}{2afokmtOQGGzfFLz3v4_Bg}{Lo4izLlTQqSMfbrf21aAkw}{192.168.0.36}{192.168.0.36:9301}{di}{xpack.installed=true},{data18b}{5HTiO0Z8RM2tRi0kaNQQ8w}{O-T8IWgXQIekz9QHQyKy4g}{192.168.0.38}{192.168.0.38:9301}{di}{xpack.installed=true},{data11b}{6FVMYvwiQD-W6fpHtVXPfg}{dAQSQFZMSjKq6kxq8cc98g}{192.168.0.31}{192.168.0.31:9301}{di}{xpack.installed=true},{data13b}{mAQ5jgAMQjaYzwpVE83kcg}{LBsGSuLJR5Kg8iWCA-SX1A}{192.168.0.33}{192.168.0.33:9301}{di}{xpack.installed=true},{data10b}{WBnLOtGgS6KXAV8FsiFYjg}{D1xTZOswTAiMc3h1YMae3Q}{192.168.0.30}{192.168.0.30:9301}{di}{xpack.installed=true},{data07}{uYjW78V_QAaAQPHjUSNjjQ}{ZA9njwKuRgOEdIllyccwqQ}{192.168.0.27}{192.168.0.27:9300}{di}{xpack.installed=true},{data17b}{7uaz7rwQRg-UQQ4WLhclBg}{SXb-eNTSTP-9-dO7pgnEfQ}{192.168.0.37}{192.168.0.37:9301}{di}{xpack.installed=true},{data17a}{x-YWJyTxRoCh8Lf4oS0qWQ}{MqzCX5bVRCa7MWZ3e7CtZg}{192.168.0.37}{192.168.0.37:9300}{di}{xpack.installed=true},{data09}{dN08oWhmSNObQgeb9xH_-A}{9aLNT59cQZutpH2xd1LzTQ}{192.168.0.29}{192.168.0.29:9300}{di}{xpack.installed=true},{data06}{NqAsywRoTVSuqTxD8cmq3A}{ffiDh9EhT0OpZ0Suj4hHcQ}{192.168.0.26}{192.168.0.26:9300}{di}{xpack.installed=true},{data18a}{_MeUtbWCQK6WAcOWs6L4fQ}{qjYEjHO0Qp-pysLXoM8iYA}{192.168.0.38}{192.168.0.38:9300}{di}{xpack.installed=true},{master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true},{master0}{xexrc0uXQUWgwFDWEpAouA}{2uUAYysWQ8-1HDcE4lAqZw}{192.168.0.10}{192.168.0.10:9300}{im}{xpack.installed=true},{data12a}{22tiK4-STyC2XRJ01nWQ3A}{rKiBCqOFTGKTBesQCzI7aw}{192.168.0.32}{192.168.0.32:9300}{di}{xpack.installed=true},{data08}{GIaGadJ0RhyBUePNzWpBdA}{fz0VawBhT_6hIre0uymIfw}{192.168.0.28}{192.168.0.28:9300}{di}{xpack.installed=true},{data15a}{0sFGQdJwSNermm8lSE7v6w}{TU-BKgdwQO2xOnhMx6o-ow}{192.168.0.35}{192.168.0.35:9300}{di}{xpack.installed=true},{data04}{M7yBnQeXTwCohL8IWN4PBg}{HRi5FXR0S_m2YlWM5zAMCw}{192.168.0.24}{192.168.0.24:9300}{di}{xpack.installed=true},{data12b}{Fs5fJqYNSLOPK7OVW7qaYQ}{2SeThcYbQ5uUG6c7JyP_Qg}{192.168.0.32}{192.168.0.32:9301}{di}{xpack.installed=true},{data13a}{vF-2rZ1ZTuG8rcYDTScDbQ}{NY6x69ejT9ehohT-uXKyww}{192.168.0.33}{192.168.0.33:9300}{di}{xpack.installed=true}}, term: 49, version: 2131760, reason: ApplyCommitRequest{term=49, version=2131760, sourceNode={master2}{RBMXwoyITeqpwJlY7VFFhA}{a7xhJcj7R46HQAm3OcmNJw}{192.168.0.12}{192.168.0.12:9300}{im}{xpack.installed=true}}
[2021-05-03T16:09:09,299][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.cluster_concurrent_rebalance] from [2] to [20]
[2021-05-03T16:09:09,299][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.same_shard.host] from [false] to [true]
[2021-05-03T16:09:09,300][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.low] from [85%] to [150gb]
[2021-05-03T16:09:09,300][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.high] from [90%] to [100gb]
[2021-05-03T16:09:09,300][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.flood_stage] from [95%] to [50gb]
[2021-05-03T16:09:09,300][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.node_initial_primaries_recoveries] from [4] to [16]
[2021-05-03T16:09:09,300][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.node_concurrent_incoming_recoveries] from [2] to [20]
[2021-05-03T16:09:09,301][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.node_concurrent_outgoing_recoveries] from [2] to [20]
[2021-05-03T16:09:09,301][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [xpack.monitoring.elasticsearch.collection.enabled] from [true] to [false]
[2021-05-03T16:09:09,301][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.low] from [85%] to [150gb]
[2021-05-03T16:09:09,301][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.high] from [90%] to [100gb]
[2021-05-03T16:09:09,302][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [cluster.routing.allocation.disk.watermark.flood_stage] from [95%] to [50gb]
[2021-05-03T16:09:09,302][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [search.max_buckets] from [10000] to [20000]
[2021-05-03T16:09:09,302][INFO ][o.e.c.s.ClusterSettings ] [master1] updating [indices.recovery.max_bytes_per_sec] from [40mb] to [120mb]
[2021-05-03T16:09:09,981][INFO ][o.e.l.LicenseService ] [master1] license [c01f4c23-89af-4849-841d-675391762534] mode [basic] - valid
[2021-05-03T16:09:10,006][INFO ][o.e.h.AbstractHttpServerTransport] [master1] publish_address {192.168.0.11:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}, {192.168.0.11:9200}
[2021-05-03T16:09:10,007][INFO ][o.e.n.Node ] [master1] started
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment