es12.log

{"type": "server", "timestamp": "2022-01-14T16:36:45,340Z", "level": "WARN", "component": "o.e.m.j.JvmGcMonitorService", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[gc][old][2559366][255519] duration [34.3s], collections [2]/[34.8s], total [34.3s]/[4.3h], memory [22gb]->[19.6gb]/[24gb], all_pools {[young] [494.4mb]->[4.5mb]/[1.1gb]}{[survivor] [149.7mb]->[0b]/[149.7mb]}{[old] [21.3gb]->[19.6gb]/[22.7gb]}", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }
{"type": "server", "timestamp": "2022-01-14T16:36:45,340Z", "level": "WARN", "component": "o.e.m.j.JvmGcMonitorService", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[gc][2559366] overhead, spent [34.4s] collecting in the last [34.8s]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }
{"type": "server", "timestamp": "2022-01-14T16:36:45,358Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[samples-2022.01.14][2]] failed to perform indices:data/write/bulk[s] on replica [samples-2022.01.14][2], node[nE9cShvvT3aHrmyx1uhcuw], [R], s[STARTED], a[id=n08RP453TWOXR87KeEle-g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [samples-2022.01.14][2] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,389Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,404Z", "level": "WARN", "component": "o.e.c.a.s.ShardStateAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "unexpected failure while sending request [internal:cluster/shard/failure] to [{es12-m2-c.FQDN}{0e7F6OGiSey77QlnY94X8Q}{VTB-GAw6Twu9RPmlnT3Mvg}{10.48.163.42}{10.48.163.42:9300}{m}{aws_az=us-west-1c, xpack.installed=true}] for shard entry [shard id [[samples-2022.01.14][2]], allocation id [n08RP453TWOXR87KeEle-g], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [samples-2022.01.14][2], node[nE9cShvvT3aHrmyx1uhcuw], [R], s[STARTED], a[id=n08RP453TWOXR87KeEle-g]], failure [RemoteTransportException[[es12-d-81986adb7ccc8d1a1ca093a2.FQDN][10.48.124.56:9300][indices:data/write/bulk[s][r]]]; nested: IllegalStateException[[samples-2022.01.14][2] operation primary term [1] is too old (current [2])]; ], markAsStale [true]]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: org.elasticsearch.cluster.action.shard.ShardStateAction$NoLongerPrimaryShardException: primary term [1] did not match current primary term [2]",
{"type": "server", "timestamp": "2022-01-14T16:36:45,405Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,405Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,416Z", "level": "WARN", "component": "o.e.c.a.s.ShardStateAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "unexpected failure while sending request [internal:cluster/shard/failure] to [{es12-m2-c.FQDN}{0e7F6OGiSey77QlnY94X8Q}{VTB-GAw6Twu9RPmlnT3Mvg}{10.48.163.42}{10.48.163.42:9300}{m}{aws_az=us-west-1c, xpack.installed=true}] for shard entry [shard id [[checkin-2022.01.14][4]], allocation id [hLt1cNuBTHmFMVCNXWVV2g], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]], failure [RemoteTransportException[[es12-d-f1ba7b3d91e22c6ca475c477.FQDN][10.48.122.104:9300][indices:data/write/bulk[s][r]]]; nested: IllegalStateException[[checkin-2022.01.14][4] operation primary term [1] is too old (current [2])]; ], markAsStale [true]]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: org.elasticsearch.cluster.action.shard.ShardStateAction$NoLongerPrimaryShardException: primary term [1] did not match current primary term [2]",
{"type": "server", "timestamp": "2022-01-14T16:36:45,414Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,433Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,455Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,483Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,502Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,509Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[[checkin-2022.01.14][4]] failed to perform indices:data/write/bulk[s] on replica [checkin-2022.01.14][4], node[dkm7td-dR3eCSGtJaBAkhQ], [R], s[STARTED], a[id=hLt1cNuBTHmFMVCNXWVV2g]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: java.lang.IllegalStateException: [checkin-2022.01.14][4] operation primary term [1] is too old (current [2])",
{"type": "server", "timestamp": "2022-01-14T16:36:45,598Z", "level": "WARN", "component": "o.e.i.e.Engine", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": " [samples-2022.01.14][2] failed engine [primary shard [[samples-2022.01.14][2], node[b7VHYPmtQk6RTrRLkjCTKA], [P], s[STARTED], a[id=2YBl_qjxRoKqssRPvU4QNw]] was demoted while failing replica shard]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
{"type": "server", "timestamp": "2022-01-14T16:36:45,610Z", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[samples-2022.01.14][2] marking and sending shard failed due to [shard failure, reason [primary shard [[samples-2022.01.14][2], node[b7VHYPmtQk6RTrRLkjCTKA], [P], s[STARTED], a[id=2YBl_qjxRoKqssRPvU4QNw]] was demoted while failing replica shard]]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
{"type": "server", "timestamp": "2022-01-14T16:36:45,907Z", "level": "WARN", "component": "o.e.i.e.Engine", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": " [checkin-2022.01.14][4] failed engine [primary shard [[checkin-2022.01.14][4], node[b7VHYPmtQk6RTrRLkjCTKA], [P], s[STARTED], a[id=RKVterAGSg-H1tp9wymiRw]] was demoted while failing replica shard]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
{"type": "server", "timestamp": "2022-01-14T16:36:45,912Z", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "[checkin-2022.01.14][4] marking and sending shard failed due to [shard failure, reason [primary shard [[checkin-2022.01.14][4], node[b7VHYPmtQk6RTrRLkjCTKA], [P], s[STARTED], a[id=RKVterAGSg-H1tp9wymiRw]] was demoted while failing replica shard]]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
{"type": "server", "timestamp": "2022-01-14T16:36:45,912Z", "level": "WARN", "component": "o.e.i.e.Engine", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": " [checkin-2022.01.14][4] tried to fail engine but engine is already failed. ignoring. [primary shard [[checkin-2022.01.14][4], node[b7VHYPmtQk6RTrRLkjCTKA], [P], s[STARTED], a[id=RKVterAGSg-H1tp9wymiRw]] was demoted while failing replica shard]", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
{"type": "server", "timestamp": "2022-01-14T16:36:47,357Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "master node [{es12-m2-c.FQDN}{0e7F6OGiSey77QlnY94X8Q}{VTB-GAw6Twu9RPmlnT3Mvg}{10.48.163.42}{10.48.163.42:9300}{m}{aws_az=us-west-1c, xpack.installed=true}] failed, restarting discovery", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA" , 
"Caused by: org.elasticsearch.transport.RemoteTransportException: [es12-m2-c.FQDN][10.48.163.42:9300][internal:coordination/fault_detection/leader_check]",
"Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: rejecting leader check since [{es12-d-d542251dda0867a36e266268.FQDN}{b7VHYPmtQk6RTrRLkjCTKA}{Z4psOW_0RFGmgTx9WG2wig}{10.48.183.179}{10.48.183.179:9300}{di}{aws_az=us-west-1c, xpack.installed=true}] has been removed from the cluster",
{"type": "server", "timestamp": "2022-01-14T16:36:47,367Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "master node changed {previous [{es12-m2-c.FQDN}{0e7F6OGiSey77QlnY94X8Q}{VTB-GAw6Twu9RPmlnT3Mvg}{10.48.163.42}{10.48.163.42:9300}{m}{aws_az=us-west-1c, xpack.installed=true}], current []}, term: 10, version: 7746850, reason: becoming candidate: onLeaderFailure", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }
{"type": "server", "timestamp": "2022-01-14T16:36:47,474Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "master node changed {previous [], current [{es12-m2-c.FQDN}{0e7F6OGiSey77QlnY94X8Q}{VTB-GAw6Twu9RPmlnT3Mvg}{10.48.163.42}{10.48.163.42:9300}{m}{aws_az=us-west-1c, xpack.installed=true}]}, term: 10, version: 7746855, reason: ApplyCommitRequest{term=10, version=7746855, sourceNode={es12-m2-c.FQDN}{0e7F6OGiSey77QlnY94X8Q}{VTB-GAw6Twu9RPmlnT3Mvg}{10.48.163.42}{10.48.163.42:9300}{m}{aws_az=us-west-1c, xpack.installed=true}}", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }
{"type": "server", "timestamp": "2022-01-14T16:36:49,379Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "no known master node, scheduling a retry", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }
{"type": "server", "timestamp": "2022-01-14T16:36:50,800Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "no known master node, scheduling a retry", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }
{"type": "server", "timestamp": "2022-01-14T16:36:51,249Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es12", "node.name": "es12-d-d542251dda0867a36e266268.FQDN", "message": "no known master node, scheduling a retry", "cluster.uuid": "eOem6Y73TvmHw8EtD-IBtw", "node.id": "b7VHYPmtQk6RTrRLkjCTKA"  }