-
-
Save vibgy/bfb427d01b0ac7c782ce8dcca4331249 to your computer and use it in GitHub Desktop.
master1
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| OpenDistro for Elasticsearch Security Demo Installer | |
| ** Warning: Do not use on production or public reachable systems ** | |
| Basedir: /usr/share/elasticsearch | |
| Elasticsearch install type: rpm/deb on CentOS Linux release 7.9.2009 (Core) | |
| Elasticsearch config dir: /usr/share/elasticsearch/config | |
| Elasticsearch config file: /usr/share/elasticsearch/config/elasticsearch.yml | |
| Elasticsearch bin dir: /usr/share/elasticsearch/bin | |
| Elasticsearch plugins dir: /usr/share/elasticsearch/plugins | |
| Elasticsearch lib dir: /usr/share/elasticsearch/lib | |
| Detected Elasticsearch Version: x-content-7.10.2 | |
| Detected Open Distro Security Version: 1.13.1.0 | |
| /usr/share/elasticsearch/config/elasticsearch.yml seems to be already configured for Security. Quit. | |
| {"timestamp":"2022-02-25T18:04:37,424Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"version[7.10.2], pid[1], build[oss/tar/747e1cc71def077253878a59143c1f785afa92b9/2021-01-13T00:42:12.435326Z], OS[Linux/5.4.104+/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/15.0.1/15.0.1+9]"} | |
| {"timestamp":"2022-02-25T18:04:37,426Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"JVM home [/opt/jdk], using bundled JDK [false]"} | |
| {"timestamp":"2022-02-25T18:04:37,427Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/elasticsearch-11510377505659667060, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.security.policy=file:///usr/share/elasticsearch/performance-analyzer-rca/pa_config/es_security.policy, -Dclk.tck=100, -Djdk.attach.allowAttachSelf=true, -Des.cgroups.hierarchy.override=/, -Xms3g, -Xmx3g, -Dlog4j2.formatMsgNoLookups=true, -XX:MaxDirectMemorySize=1610612736, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=oss, -Des.distribution.type=tar, -Des.bundled_jdk=true]"} | |
| {"timestamp":"2022-02-25T18:04:38,432Z","level":"INFO","component":"c.a.o.e.p.c.PluginSettings","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Config: metricsLocation: /dev/shm/performanceanalyzer/, metricsDeletionInterval: 1, httpsEnabled: false, cleanup-metrics-db-files: true, batch-metrics-retention-period-minutes: 7, rpc-port: 9650, webservice-port 9600"} | |
| {"timestamp":"2022-02-25T18:04:38,826Z","level":"INFO","component":"c.a.o.s.s.t.OpenDistroSSLConfig","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"SSL dual mode is disabled"} | |
| {"timestamp":"2022-02-25T18:04:38,827Z","level":"INFO","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"ES Config path is /usr/share/elasticsearch/config"} | |
| {"timestamp":"2022-02-25T18:04:39,048Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"JVM supports TLSv1.3"} | |
| {"timestamp":"2022-02-25T18:04:39,050Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Config directory is /usr/share/elasticsearch/config/, from there the key- and truststore files are resolved relatively"} | |
| {"timestamp":"2022-02-25T18:04:39,378Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"TLS Transport Client Provider : JDK"} | |
| {"timestamp":"2022-02-25T18:04:39,378Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"TLS Transport Server Provider : JDK"} | |
| {"timestamp":"2022-02-25T18:04:39,378Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"TLS HTTP Provider : null"} | |
| {"timestamp":"2022-02-25T18:04:39,378Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Enabled TLS protocols for transport layer : [TLSv1.3, TLSv1.2, TLSv1.1]"} | |
| {"timestamp":"2022-02-25T18:04:39,379Z","level":"INFO","component":"c.a.o.s.s.DefaultOpenDistroSecurityKeyStore","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Enabled TLS protocols for HTTP layer : []"} | |
| {"timestamp":"2022-02-25T18:04:39,601Z","level":"INFO","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Clustername: elasticsearch-logging-us-west1"} | |
| {"timestamp":"2022-02-25T18:04:39,607Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Directory /usr/share/elasticsearch/config has insecure file permissions (should be 0700)"} | |
| {"timestamp":"2022-02-25T18:04:39,607Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/log4j2.properties has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,608Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/elasticsearch.yml has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,608Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Directory /usr/share/elasticsearch/config/certificates has insecure file permissions (should be 0700)"} | |
| {"timestamp":"2022-02-25T18:04:39,608Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/..data has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,608Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/node.pem has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,609Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/node.key has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,609Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/admin.pem has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,609Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/root-ca.pem has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,609Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Directory /usr/share/elasticsearch/config/certificates/..2022_02_25_18_04_12.891935657 has insecure file permissions (should be 0700)"} | |
| {"timestamp":"2022-02-25T18:04:39,610Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/..2022_02_25_18_04_12.891935657/node.pem has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,610Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/..2022_02_25_18_04_12.891935657/node.key has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,610Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/..2022_02_25_18_04_12.891935657/admin.pem has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,610Z","level":"WARN","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"File /usr/share/elasticsearch/config/certificates/..2022_02_25_18_04_12.891935657/root-ca.pem has insecure file permissions (should be 0600)"} | |
| {"timestamp":"2022-02-25T18:04:39,903Z","level":"INFO","component":"c.a.o.j.JobSchedulerPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Loaded scheduler extension: opendistro-index-management, index: .opendistro-ism-config"} | |
| {"timestamp":"2022-02-25T18:04:39,926Z","level":"INFO","component":"c.a.o.j.JobSchedulerPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Loaded scheduler extension: reports-scheduler, index: .opendistro-reports-definitions"} | |
| {"timestamp":"2022-02-25T18:04:39,931Z","level":"INFO","component":"c.a.o.j.JobSchedulerPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Loaded scheduler extension: opendistro_anomaly_detector, index: .opendistro-anomaly-detector-jobs"} | |
| {"timestamp":"2022-02-25T18:04:39,933Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [aggs-matrix-stats]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [analysis-common]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [geo]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [ingest-common]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [ingest-geoip]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [ingest-user-agent]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [kibana]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [lang-expression]"} | |
| {"timestamp":"2022-02-25T18:04:39,934Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [lang-mustache]"} | |
| {"timestamp":"2022-02-25T18:04:39,935Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [lang-painless]"} | |
| {"timestamp":"2022-02-25T18:04:39,935Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [mapper-extras]"} | |
| {"timestamp":"2022-02-25T18:04:39,935Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [parent-join]"} | |
| {"timestamp":"2022-02-25T18:04:39,935Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [percolator]"} | |
| {"timestamp":"2022-02-25T18:04:39,935Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [rank-eval]"} | |
| {"timestamp":"2022-02-25T18:04:39,935Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [reindex]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [repository-url]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded module [transport-netty4]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [mapper-size]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-alerting]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-anomaly-detection]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-asynchronous-search]"} | |
| {"timestamp":"2022-02-25T18:04:39,936Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-index-management]"} | |
| {"timestamp":"2022-02-25T18:04:39,937Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-job-scheduler]"} | |
| {"timestamp":"2022-02-25T18:04:39,937Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-knn]"} | |
| {"timestamp":"2022-02-25T18:04:39,937Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-performance-analyzer]"} | |
| {"timestamp":"2022-02-25T18:04:39,937Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-reports-scheduler]"} | |
| {"timestamp":"2022-02-25T18:04:39,937Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro-sql]"} | |
| {"timestamp":"2022-02-25T18:04:39,937Z","level":"INFO","component":"o.e.p.PluginsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"loaded plugin [opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:04:39,967Z","level":"INFO","component":"o.e.e.NodeEnvironment","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sdb)]], net usable_space [195.7gb], net total_space [195.8gb], types [ext4]"} | |
| {"timestamp":"2022-02-25T18:04:39,968Z","level":"INFO","component":"o.e.e.NodeEnvironment","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"heap size [3gb], compressed ordinary object pointers [true]"} | |
| {"timestamp":"2022-02-25T18:04:40,049Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node name [elasticsearch-logging-master-us-west1-a-1], node ID [4msBo_zpQD27irf5y4WRJw], cluster name [elasticsearch-logging-us-west1], roles [master]"} | |
| {"timestamp":"2022-02-25T18:04:42,092Z","level":"DEPRECATION","component":"o.e.d.c.r.OperationRouting","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"searches will not be routed based on awareness attributes starting in version 8.0.0; to opt into this behaviour now please set the system property [es.search.ignore_awareness_attributes] to [true]"} | |
| {"timestamp":"2022-02-25T18:04:43,117Z","level":"WARN","component":"c.a.o.s.c.Salt","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"If you plan to use field masking pls configure compliance salt e1ukloTsQlOgPquJ to be a random string of 16 chars length identical on all nodes"} | |
| {"timestamp":"2022-02-25T18:04:43,128Z","level":"ERROR","component":"c.a.o.s.a.s.SinkProvider","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Default endpoint could not be created, auditlog will not work properly."} | |
| {"timestamp":"2022-02-25T18:04:43,129Z","level":"WARN","component":"c.a.o.s.a.r.AuditMessageRouter","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"No default storage available, audit log may not work properly. Please check configuration."} | |
| {"timestamp":"2022-02-25T18:04:43,129Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Message routing enabled: false"} | |
| {"timestamp":"2022-02-25T18:04:43,171Z","level":"INFO","component":"c.a.o.s.f.OpenDistroSecurityFilter","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"<NONE> indices are made immutable."} | |
| {"timestamp":"2022-02-25T18:04:43,412Z","level":"INFO","component":"c.a.o.a.b.ADCircuitBreakerService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Registered memory breaker."} | |
| {"timestamp":"2022-02-25T18:04:43,601Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,646Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,647Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,647Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,647Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,648Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,648Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,648Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,648Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,648Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,649Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,649Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,649Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,652Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,652Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,652Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,654Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,655Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,655Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,655Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,655Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,655Z","level":"WARN","component":"c.a.o.e.p.c.ThreadPoolMetricsCollector","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Fail to read queue capacity via reflection"} | |
| {"timestamp":"2022-02-25T18:04:43,670Z","level":"INFO","component":"o.e.t.NettyAllocator","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=512kb, suggested_max_allocation_size=512kb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=2mb}]"} | |
| {"timestamp":"2022-02-25T18:04:43,737Z","level":"INFO","component":"o.e.d.DiscoveryModule","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"using discovery type [zen] and seed hosts providers [settings]"} | |
| {"timestamp":"2022-02-25T18:04:44,047Z","level":"WARN","component":"o.e.g.DanglingIndicesState","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually"} | |
| {"timestamp":"2022-02-25T18:04:44,274Z","level":"INFO","component":"c.a.o.e.p.h.c.PerformanceAnalyzerConfigAction","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"PerformanceAnalyzer Enabled: true"} | |
| {"timestamp":"2022-02-25T18:04:44,352Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"initialized"} | |
| {"timestamp":"2022-02-25T18:04:44,352Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"starting ..."} | |
| {"timestamp":"2022-02-25T18:04:44,442Z","level":"INFO","component":"o.e.t.TransportService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"publish_address {10.144.12.40:9300}, bound_addresses {0.0.0.0:9300}"} | |
| {"timestamp":"2022-02-25T18:04:44,831Z","level":"INFO","component":"o.e.b.BootstrapChecks","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"bound or publishing to a non-loopback address, enforcing bootstrap checks"} | |
| {"timestamp":"2022-02-25T18:04:44,833Z","level":"INFO","component":"o.e.c.c.Coordinator","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"cluster UUID [ViiZxJxCS66YGlFeY4DTUA]"} | |
| {"timestamp":"2022-02-25T18:04:46,206Z","level":"INFO","component":"o.e.c.c.JoinHelper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to join {elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard} with JoinRequest{sourceNode={elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}, minimumTerm=318, optionalJoin=Optional[Join{term=319, lastAcceptedTerm=318, lastAcceptedVersion=41061873, sourceNode={elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}, targetNode={elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}}]}", "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-logging-master-us-west1-a-1][10.144.12.40:9300][internal:cluster/coordination/join]","Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}","at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:464) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:533) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:496) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:377) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:364) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:483) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:140) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistro.elasticsearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:49) [opendistro-performance-analyzer-1.13.0.0.jar:1.13.0.0]","at com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:182) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:170) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:92) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:639) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:124) [opendistro-index-management-1.13.2.0.jar:1.13.2.0]","at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:72) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:809) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:743) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.RemoteTransportException: [elasticsearch-logging-master-us-west1-a-1][10.144.12.40:9300][internal:cluster/coordination/join] | |
| Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard} | |
| at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:464) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:533) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:496) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:377) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:364) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:483) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:140) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistro.elasticsearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:49) [opendistro-performance-analyzer-1.13.0.0.jar:1.13.0.0] | |
| at com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceivedDecorate(OpenDistroSecuritySSLRequestHandler.java:182) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityRequestHandler.messageReceivedDecorate(OpenDistroSecurityRequestHandler.java:170) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at com.amazon.opendistroforelasticsearch.security.ssl.transport.OpenDistroSecuritySSLRequestHandler.messageReceived(OpenDistroSecuritySSLRequestHandler.java:92) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at com.amazon.opendistroforelasticsearch.security.OpenDistroSecurityPlugin$7$1.messageReceived(OpenDistroSecurityPlugin.java:639) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:124) [opendistro-index-management-1.13.2.0.jar:1.13.2.0] | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:72) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:809) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:743) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T18:04:46,282Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"elected-as-master ([2] nodes joined)[{elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} elect leader, {elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 320, version: 41061874, delta: master node changed {previous [], current [{elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}]}, added {{elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:04:46,490Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"master node changed {previous [], current [{elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}]}, added {{elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41061874, reason: Publication{term=320, version=41061874}"} | |
| {"timestamp":"2022-02-25T18:04:46,502Z","level":"WARN","component":"c.a.o.e.p.c.s.h.ConfigOverridesClusterSettingHandler","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Config override setting update called with empty string. Ignoring."} | |
| {"timestamp":"2022-02-25T18:04:46,554Z","level":"INFO","component":"o.e.h.AbstractHttpServerTransport","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"publish_address {10.144.12.40:9200}, bound_addresses {0.0.0.0:9200}"} | |
| {"timestamp":"2022-02-25T18:04:46,554Z","level":"INFO","component":"o.e.n.Node","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"started"} | |
| {"timestamp":"2022-02-25T18:04:46,555Z","level":"INFO","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Node started"} | |
| {"timestamp":"2022-02-25T18:04:46,555Z","level":"INFO","component":"c.a.o.s.c.ConfigurationRepository","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Will not attempt to create index .opendistro_security and default configs if they are absent. Use securityadmin to initialize cluster"} | |
| {"timestamp":"2022-02-25T18:04:46,556Z","level":"INFO","component":"c.a.o.s.c.ConfigurationRepository","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Background init thread started. Install default config?: false"} | |
| {"timestamp":"2022-02-25T18:04:46,557Z","level":"INFO","component":"c.a.o.s.OpenDistroSecurityPlugin","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"0 Open Distro Security modules loaded so far: []"} | |
| WARNING: An illegal reflective access operation has occurred | |
| WARNING: Illegal reflective access by com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics (file:/usr/share/elasticsearch/plugins/opendistro-performance-analyzer/opendistro-performance-analyzer-1.13.0.0.jar) to field java.util.concurrent.ThreadPoolExecutor.workers | |
| WARNING: Please consider reporting this to the maintainers of com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.MasterServiceEventMetrics | |
| WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations | |
| WARNING: All illegal access operations will be denied in a future release | |
| {"timestamp":"2022-02-25T18:04:46,822Z","level":"INFO","component":"o.e.c.s.ClusterSettings","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating [cluster.metadata.perf_analyzer.state] from [] to [1]"} | |
| {"timestamp":"2022-02-25T18:04:46,823Z","level":"INFO","component":"o.e.c.s.ClusterSettings","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating [cluster.routing.allocation.balance.index] from [0.55] to [0.9]"} | |
| {"timestamp":"2022-02-25T18:04:46,823Z","level":"INFO","component":"o.e.c.s.ClusterSettings","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating [cluster.routing.allocation.balance.shard] from [0.45] to [0.1]"} | |
| {"timestamp":"2022-02-25T18:04:46,824Z","level":"INFO","component":"o.e.c.s.ClusterSettings","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating [cluster.metadata.perf_analyzer.state] from [0] to [1]"} | |
| {"timestamp":"2022-02-25T18:04:46,848Z","level":"INFO","component":"o.e.g.GatewayService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"recovered [142] indices into cluster_state"} | |
| {"timestamp":"2022-02-25T18:04:46,876Z","level":"ERROR","component":"c.a.o.i.i.ManagedIndexCoordinator","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"get managed-index failed: NoShardAvailableActionException[No shard available for [org.elasticsearch.action.get.MultiGetShardRequest@519338db]]"} | |
| {"timestamp":"2022-02-25T18:04:46,914Z","level":"ERROR","component":"c.a.o.i.i.ManagedIndexCoordinator","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Failed to get ISM templates: Failed to execute phase [query], all shards failed"} | |
| {"timestamp":"2022-02-25T18:04:47,343Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [0] for indices [.opendistro_security, .kibana_task_manager_1]"} | |
| {"timestamp":"2022-02-25T18:04:47,345Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{hwLc07R-T1aTSifd8uxYIg}{10.144.38.15}{10.144.38.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader], term: 320, version: 41061876, delta: added {{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{hwLc07R-T1aTSifd8uxYIg}{10.144.38.15}{10.144.38.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:04:49,757Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{hwLc07R-T1aTSifd8uxYIg}{10.144.38.15}{10.144.38.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41061876, reason: Publication{term=320, version=41061876}"} | |
| {"timestamp":"2022-02-25T18:04:49,766Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [1] for indices [.kibana_task_manager_1]"} | |
| {"timestamp":"2022-02-25T18:04:49,767Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [2] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:04:49,769Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-coordinator-us-west1-a-0}{rvcds710SVeRHTMwCPOfsg}{VwFmP7gbTriKAcOjdxWNmA}{10.144.29.7}{10.144.29.7:9300}{region=us-west1, zone=us-west1-a, storageclass=none} join existing leader, {elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{IsZJxFPSTuSfBWECEdKSFQ}{10.144.32.15}{10.144.32.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader, {elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{vtauUOL1RPCWydRZa7xbBQ}{10.144.22.18}{10.144.22.18:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader], term: 320, version: 41061877, delta: added {{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{IsZJxFPSTuSfBWECEdKSFQ}{10.144.32.15}{10.144.32.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard},{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{vtauUOL1RPCWydRZa7xbBQ}{10.144.22.18}{10.144.22.18:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard},{elasticsearch-logging-coordinator-us-west1-a-0}{rvcds710SVeRHTMwCPOfsg}{VwFmP7gbTriKAcOjdxWNmA}{10.144.29.7}{10.144.29.7:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}"} | |
| {"timestamp":"2022-02-25T18:04:53,219Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{IsZJxFPSTuSfBWECEdKSFQ}{10.144.32.15}{10.144.32.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard},{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{vtauUOL1RPCWydRZa7xbBQ}{10.144.22.18}{10.144.22.18:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard},{elasticsearch-logging-coordinator-us-west1-a-0}{rvcds710SVeRHTMwCPOfsg}{VwFmP7gbTriKAcOjdxWNmA}{10.144.29.7}{10.144.29.7:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}, term: 320, version: 41061877, reason: Publication{term=320, version=41061877}"} | |
| {"timestamp":"2022-02-25T18:04:53,627Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader], term: 320, version: 41061880, delta: added {{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:04:54,895Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41061880, reason: Publication{term=320, version=41061880}"} | |
| {"timestamp":"2022-02-25T18:04:55,237Z","level":"INFO","component":"stdout","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[FINE] No subscribers registered for event class com.amazon.opendistroforelasticsearch.security.securityconf.DynamicConfigFactory$NodesDnModelImpl"} | |
| {"timestamp":"2022-02-25T18:04:55,237Z","level":"INFO","component":"stdout","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[FINE] No subscribers registered for event class org.greenrobot.eventbus.NoSubscriberEvent"} | |
| {"timestamp":"2022-02-25T18:04:55,238Z","level":"INFO","component":"c.a.o.s.c.ConfigurationRepository","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Hot-reloading of audit configuration is disabled. Using configuration with defaults from elasticsearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it."} | |
| {"timestamp":"2022-02-25T18:04:55,238Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing on REST API is enabled."} | |
| {"timestamp":"2022-02-25T18:04:55,238Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from REST API auditing."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing on Transport API is enabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from Transport API auditing."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing of request body is enabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Bulk requests resolution is disabled during request auditing."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Index resolution is enabled during request auditing."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Sensitive headers auditing is enabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing requests from kibanaserver users is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing of external configuration is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing of internal configuration is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing only metadata information for read request is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,239Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing will watch {} for read requests."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing read operation requests from kibanaserver users is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing only metadata information for write request is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing diffs for write requests is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing write operation requests from kibanaserver users is disabled."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Auditing will watch <NONE> for write requests."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":".opendistro_security is used as internal security index."} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.a.i.AuditLogImpl","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Internal index used for posting audit logs is null"} | |
| {"timestamp":"2022-02-25T18:04:55,240Z","level":"INFO","component":"c.a.o.s.c.ConfigurationRepository","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Node 'elasticsearch-logging-master-us-west1-a-1' initialized"} | |
| {"timestamp":"2022-02-25T18:05:00,854Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][2] marking unavailable shards as stale: [H3_UkfJKQ_eJInPm9UsMHw]"} | |
| {"timestamp":"2022-02-25T18:05:00,854Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][1] marking unavailable shards as stale: [Zf1E0CpDTA6Zfi9jUx9CKQ]"} | |
| {"timestamp":"2022-02-25T18:05:00,854Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][0] marking unavailable shards as stale: [jsoVMRn9TNiN10KIGg2Zbg]"} | |
| {"timestamp":"2022-02-25T18:05:01,713Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][0] marking unavailable shards as stale: [x4TltJCdSR2YR-JB9qStSg]"} | |
| {"timestamp":"2022-02-25T18:05:01,713Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][1] marking unavailable shards as stale: [y0Bvph-YREKFY2YgQKea8g]"} | |
| {"timestamp":"2022-02-25T18:05:01,714Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][2] marking unavailable shards as stale: [qZpjXAKPTCimLeETm7mpfw]"} | |
| {"timestamp":"2022-02-25T18:05:17,331Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-coordinator-us-west1-a-1}{Vce2y-DvS6KdkfLXKTn4mg}{-s618yM1SIWseniuyP6GgQ}{10.144.12.41}{10.144.12.41:9300}{region=us-west1, zone=us-west1-a, storageclass=none} join existing leader], term: 320, version: 41062158, delta: added {{elasticsearch-logging-coordinator-us-west1-a-1}{Vce2y-DvS6KdkfLXKTn4mg}{-s618yM1SIWseniuyP6GgQ}{10.144.12.41}{10.144.12.41:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}"} | |
| {"timestamp":"2022-02-25T18:05:20,282Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-coordinator-us-west1-a-1}{Vce2y-DvS6KdkfLXKTn4mg}{-s618yM1SIWseniuyP6GgQ}{10.144.12.41}{10.144.12.41:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}, term: 320, version: 41062158, reason: Publication{term=320, version=41062158}"} | |
| {"timestamp":"2022-02-25T18:05:29,788Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:05:34,034Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[.k8s-system-logs-000006][1]]])."} | |
| {"timestamp":"2022-02-25T18:05:44,401Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-coordinator-us-west1-a-2}{pdPTjzLlTmOpRHVy8sRo2g}{iQQdp0d5SySbQlQVn2xh9A}{10.144.28.30}{10.144.28.30:9300}{region=us-west1, zone=us-west1-a, storageclass=none} join existing leader], term: 320, version: 41062295, delta: added {{elasticsearch-logging-coordinator-us-west1-a-2}{pdPTjzLlTmOpRHVy8sRo2g}{iQQdp0d5SySbQlQVn2xh9A}{10.144.28.30}{10.144.28.30:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}"} | |
| {"timestamp":"2022-02-25T18:05:48,361Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-coordinator-us-west1-a-2}{pdPTjzLlTmOpRHVy8sRo2g}{iQQdp0d5SySbQlQVn2xh9A}{10.144.28.30}{10.144.28.30:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}, term: 320, version: 41062295, reason: Publication{term=320, version=41062295}"} | |
| {"timestamp":"2022-02-25T18:05:59,791Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:07:44,393Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.opendistro-ism-config][0]]])."} | |
| {"timestamp":"2022-02-25T18:07:44,954Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [1] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:07:44,977Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-left[{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{vtauUOL1RPCWydRZa7xbBQ}{10.144.22.18}{10.144.22.18:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} reason: disconnected], term: 320, version: 41062892, delta: removed {{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{vtauUOL1RPCWydRZa7xbBQ}{10.144.22.18}{10.144.22.18:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:07:45,567Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"removed {{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{vtauUOL1RPCWydRZa7xbBQ}{10.144.22.18}{10.144.22.18:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41062892, reason: Publication{term=320, version=41062892}"} | |
| {"timestamp":"2022-02-25T18:07:45,574Z","level":"INFO","component":"o.e.c.r.DelayedAllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"scheduling reroute for delayed shards in [59.3s] (239 delayed shards)"} | |
| {"timestamp":"2022-02-25T18:07:45,589Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][0] marking unavailable shards as stale: [mWiE4RDwRfydL-3QlAPxLw]"} | |
| {"timestamp":"2022-02-25T18:07:45,589Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][1] marking unavailable shards as stale: [elHOu4IfRteVYdm3PfVT1A]"} | |
| {"timestamp":"2022-02-25T18:07:45,589Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][2] marking unavailable shards as stale: [LVtMhhbBRxiQUSW88xj5Yw]"} | |
| {"timestamp":"2022-02-25T18:07:45,590Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][0] marking unavailable shards as stale: [hDuQf1aBQ9awMYK99Xx5rQ]"} | |
| {"timestamp":"2022-02-25T18:08:11,660Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.opendistro_security][0] marking unavailable shards as stale: [2gaXlktCTW6c_VtepY-31w]"} | |
| {"timestamp":"2022-02-25T18:08:20,383Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [2] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:08:20,383Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{5scGvXVjRkGDMs_9YOE9Og}{10.144.22.19}{10.144.22.19:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader], term: 320, version: 41062895, delta: added {{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{5scGvXVjRkGDMs_9YOE9Og}{10.144.22.19}{10.144.22.19:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:08:21,102Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-data-us-west1-a-0}{bLMdCrq4RiCvXji9j17fgg}{5scGvXVjRkGDMs_9YOE9Og}{10.144.22.19}{10.144.22.19:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41062895, reason: Publication{term=320, version=41062895}"} | |
| {"timestamp":"2022-02-25T18:08:44,979Z","level":"INFO","component":"o.e.c.r.DelayedAllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"scheduling reroute for delayed shards in [3.9m] (202 delayed shards)"} | |
| {"timestamp":"2022-02-25T18:09:44,837Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:10:28,748Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,151Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.opendistro-ism-config][0]]])."} | |
| {"timestamp":"2022-02-25T18:10:40,190Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,220Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,248Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,284Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,293Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,305Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,313Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,326Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,335Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:10:40,983Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [1] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:10:41,009Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-left[{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{IsZJxFPSTuSfBWECEdKSFQ}{10.144.32.15}{10.144.32.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} reason: disconnected], term: 320, version: 41063332, delta: removed {{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{IsZJxFPSTuSfBWECEdKSFQ}{10.144.32.15}{10.144.32.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:10:42,146Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"removed {{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{IsZJxFPSTuSfBWECEdKSFQ}{10.144.32.15}{10.144.32.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41063332, reason: Publication{term=320, version=41063332}"} | |
| {"timestamp":"2022-02-25T18:10:42,149Z","level":"INFO","component":"o.e.c.r.DelayedAllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"scheduling reroute for delayed shards in [58.8s] (240 delayed shards)"} | |
| {"timestamp":"2022-02-25T18:10:42,158Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][1] marking unavailable shards as stale: [jqIDLr-TSh655sWHDgDXdQ]"} | |
| {"timestamp":"2022-02-25T18:10:42,159Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.opendistro_security][0] marking unavailable shards as stale: [gYTYwFHhSjqiQWad8apbZg]"} | |
| {"timestamp":"2022-02-25T18:10:42,207Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][0] marking unavailable shards as stale: [s6XTZIsGQju9A5HoPDw__w]"} | |
| {"timestamp":"2022-02-25T18:10:42,208Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][2] marking unavailable shards as stale: [qn5G7JbjSLmhAm5-cSCsag]"} | |
| {"timestamp":"2022-02-25T18:10:42,267Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][2] marking unavailable shards as stale: [B-vGj__XT9e1thAftUcWJg]"} | |
| {"timestamp":"2022-02-25T18:11:16,152Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [2] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:11:16,153Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{zMLAeFuZR9uM9kcH32OJ2w}{10.144.32.16}{10.144.32.16:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader], term: 320, version: 41063336, delta: added {{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{zMLAeFuZR9uM9kcH32OJ2w}{10.144.32.16}{10.144.32.16:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:11:16,853Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-data-us-west1-a-1}{rP0pVKSsQ1e4rD-LDkMQOA}{zMLAeFuZR9uM9kcH32OJ2w}{10.144.32.16}{10.144.32.16:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41063336, reason: Publication{term=320, version=41063336}"} | |
| {"timestamp":"2022-02-25T18:11:41,007Z","level":"INFO","component":"o.e.c.r.DelayedAllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"scheduling reroute for delayed shards in [3.9m] (213 delayed shards)"} | |
| {"timestamp":"2022-02-25T18:13:28,559Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.k8s-container-logs-000093][0]]])."} | |
| {"timestamp":"2022-02-25T18:13:31,714Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [1] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:13:31,733Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-left[{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{hwLc07R-T1aTSifd8uxYIg}{10.144.38.15}{10.144.38.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} reason: disconnected], term: 320, version: 41063776, delta: removed {{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{hwLc07R-T1aTSifd8uxYIg}{10.144.38.15}{10.144.38.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:13:32,829Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"removed {{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{hwLc07R-T1aTSifd8uxYIg}{10.144.38.15}{10.144.38.15:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41063776, reason: Publication{term=320, version=41063776}"} | |
| {"timestamp":"2022-02-25T18:13:32,830Z","level":"INFO","component":"o.e.c.r.DelayedAllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"scheduling reroute for delayed shards in [58.8s] (239 delayed shards)"} | |
| {"timestamp":"2022-02-25T18:13:32,837Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][0] marking unavailable shards as stale: [5Stca9d_QeWTwXwJfvwhBw]"} | |
| {"timestamp":"2022-02-25T18:13:32,878Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][1] marking unavailable shards as stale: [TJeASA9ITI6jKr5gzsSGNQ]"} | |
| {"timestamp":"2022-02-25T18:13:32,878Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-container-logs-001527][1] marking unavailable shards as stale: [UI6uL0DNQaWkaE1zYezYnQ]"} | |
| {"timestamp":"2022-02-25T18:13:32,920Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000362][2] marking unavailable shards as stale: [aTwb40jlSxaGjHV6cJkJDA]"} | |
| {"timestamp":"2022-02-25T18:13:51,459Z","level":"WARN","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.opendistro_security][0] marking unavailable shards as stale: [z8Ao3g_dTqi0xQyViC_hNg]"} | |
| {"timestamp":"2022-02-25T18:14:27,737Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"updating number_of_replicas to [2] for indices [.opendistro_security]"} | |
| {"timestamp":"2022-02-25T18:14:27,738Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{YWthA5ChQb-0wst5CcT_jQ}{10.144.38.16}{10.144.38.16:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} join existing leader], term: 320, version: 41063781, delta: added {{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{YWthA5ChQb-0wst5CcT_jQ}{10.144.38.16}{10.144.38.16:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:14:28,500Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-data-us-west1-a-2}{uhCH0NrVTB6mdxn0ZOxFLQ}{YWthA5ChQb-0wst5CcT_jQ}{10.144.38.16}{10.144.38.16:9300}{d}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41063781, reason: Publication{term=320, version=41063781}"} | |
| {"timestamp":"2022-02-25T18:14:36,768Z","level":"INFO","component":"o.e.c.r.DelayedAllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"scheduling reroute for delayed shards in [3.9m] (217 delayed shards)"} | |
| {"timestamp":"2022-02-25T18:14:44,838Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:15:28,749Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:15:58,751Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:16:46,266Z","level":"INFO","component":"o.e.c.r.a.AllocationService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.k8s-container-logs-000093][2]]])."} | |
| {"timestamp":"2022-02-25T18:19:44,838Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:20:28,750Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:20:28,751Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:24:44,839Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:25:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:25:28,748Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:29:44,839Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:30:28,751Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:30:28,752Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:34:44,840Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:35:28,749Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:35:28,750Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:38:42,695Z","level":"INFO","component":"o.e.c.m.MetadataDeleteIndexService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.k8s-system-logs-000338/kEw2vE1uQam2IYXquTIi9w] deleting index"} | |
| {"timestamp":"2022-02-25T18:39:44,840Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:40:28,750Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:40:28,751Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:40:50,351Z","level":"INFO","component":"o.e.c.m.MetadataDeleteIndexService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.remote-logs-000030/CmGIfZFcT_uAdVhrS8E6TA] deleting index"} | |
| {"timestamp":"2022-02-25T18:41:46,181Z","level":"INFO","component":"o.e.c.m.MetadataDeleteIndexService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"[.sensor-proxy-device-logs-000616/VAotv5T3QpOw-MiWWQBryA] deleting index"} | |
| {"timestamp":"2022-02-25T18:44:44,841Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:45:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,748Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,762Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,769Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,778Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,784Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,792Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,798Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,807Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:45:28,812Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:49:44,842Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:50:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,763Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,771Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,780Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,788Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,797Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,804Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,814Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:50:28,822Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:53:29,288Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-left[{elasticsearch-logging-coordinator-us-west1-a-0}{rvcds710SVeRHTMwCPOfsg}{VwFmP7gbTriKAcOjdxWNmA}{10.144.29.7}{10.144.29.7:9300}{region=us-west1, zone=us-west1-a, storageclass=none} reason: disconnected], term: 320, version: 41064988, delta: removed {{elasticsearch-logging-coordinator-us-west1-a-0}{rvcds710SVeRHTMwCPOfsg}{VwFmP7gbTriKAcOjdxWNmA}{10.144.29.7}{10.144.29.7:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}"} | |
| {"timestamp":"2022-02-25T18:53:29,335Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"removed {{elasticsearch-logging-coordinator-us-west1-a-0}{rvcds710SVeRHTMwCPOfsg}{VwFmP7gbTriKAcOjdxWNmA}{10.144.29.7}{10.144.29.7:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}, term: 320, version: 41064988, reason: Publication{term=320, version=41064988}"} | |
| {"timestamp":"2022-02-25T18:53:29,344Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-left[{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} reason: disconnected], term: 320, version: 41064989, delta: removed {{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:53:29,343Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [1] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-2][10.144.29.9:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.144.29.9/10.144.29.9:9300","Caused by: java.net.ConnectException: Connection refused","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-2][10.144.29.9:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.144.29.9/10.144.29.9:9300 | |
| Caused by: java.net.ConnectException: Connection refused | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T18:53:29,372Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"removed {{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{ThaZrXRsQaaj89_7sLna5g}{10.144.29.9}{10.144.29.9:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 320, version: 41064989, reason: Publication{term=320, version=41064989}"} | |
| {"timestamp":"2022-02-25T18:54:29,322Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-join[{elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} join existing leader], term: 320, version: 41065038, delta: added {{elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}"} | |
| {"timestamp":"2022-02-25T18:54:31,275Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"added {{elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}}, term: 320, version: 41065038, reason: Publication{term=320, version=41065038}"} | |
| {"timestamp":"2022-02-25T18:54:44,842Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T18:54:50,276Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"node-left[{elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} reason: disconnected], term: 320, version: 41065039, delta: removed {{elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:54:50,280Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"master node changed {previous [{elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}], current []}, term: 320, version: 41065038, reason: becoming candidate: Publication.onCompletion(false)"} | |
| {"timestamp":"2022-02-25T18:54:50,280Z","level":"WARN","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failing [node-left[{elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} reason: disconnected]]: failed to commit cluster state version [41065039]", "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed","at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1467) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:88) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:224) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1390) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:173) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Publication.start(Publication.java:72) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1115) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:268) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum","at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:171) ~[elasticsearch-7.10.2.jar:7.10.2]","... 14 more"]} | |
| org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed | |
| at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1467) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:88) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:224) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1390) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:173) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Publication.start(Publication.java:72) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1115) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:268) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum | |
| at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:171) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| ... 14 more | |
| {"timestamp":"2022-02-25T18:54:50,282Z","level":"ERROR","component":"o.e.c.c.Coordinator","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"unexpected failure during [node-left]", "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed","at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1467) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:88) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:224) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1390) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:173) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Publication.start(Publication.java:72) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1115) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:268) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum","at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:171) ~[elasticsearch-7.10.2.jar:7.10.2]","... 14 more"]} | |
| org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed | |
| at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1467) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:88) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:224) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1390) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:173) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Publication.start(Publication.java:72) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1115) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:268) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum | |
| at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:171) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| ... 14 more | |
| {"timestamp":"2022-02-25T18:54:50,285Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [1] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.ConnectException: Connection refused","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.ConnectException: Connection refused | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T18:54:51,368Z","level":"INFO","component":"o.e.c.s.MasterService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"elected-as-master ([2] nodes joined)[{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{mF0w8v2tTcWWxxKAdWYphQ}{10.144.51.11}{10.144.51.11:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} elect leader, {elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 321, version: 41065039, delta: master node changed {previous [], current [{elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}]}, added {{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{mF0w8v2tTcWWxxKAdWYphQ}{10.144.51.11}{10.144.51.11:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}"} | |
| {"timestamp":"2022-02-25T18:55:01,375Z","level":"INFO","component":"o.e.c.c.C.CoordinatorPublication","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"after [10s] publication of cluster state version [41065039] is still waiting for {elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{mF0w8v2tTcWWxxKAdWYphQ}{10.144.51.11}{10.144.51.11:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} [SENT_APPLY_COMMIT]"} | |
| {"timestamp":"2022-02-25T18:55:21,375Z","level":"INFO","component":"o.e.c.s.ClusterApplierService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"master node changed {previous [], current [{elasticsearch-logging-master-us-west1-a-1}{4msBo_zpQD27irf5y4WRJw}{bAxw42i8Sgmghgdqc2oqDA}{10.144.12.40}{10.144.12.40:9300}{m}{zone=us-west1-a, region=us-west1, storageclass=pd-standard}]}, added {{elasticsearch-logging-master-us-west1-a-2}{VN24LF_rQb-krhtlB8gTnw}{mF0w8v2tTcWWxxKAdWYphQ}{10.144.51.11}{10.144.51.11:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard}}, term: 321, version: 41065039, reason: Publication{term=321, version=41065039}"} | |
| {"timestamp":"2022-02-25T18:55:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,748Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,827Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,834Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,843Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,850Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,859Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,865Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,872Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:55:28,876Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T18:58:44,478Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [7] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_timeout[30s]","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_timeout[30s] | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T18:59:44,842Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:00:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,758Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,763Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,769Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,773Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,780Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,784Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,790Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:00:28,794Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:02:44,494Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [13] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_timeout[30s]","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_timeout[30s] | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:04:44,843Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:05:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,759Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,763Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,769Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,773Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,778Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,782Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,788Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:05:28,792Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:06:44,509Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [19] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_timeout[30s]","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_timeout[30s] | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:09:44,843Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:09:50,378Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [25] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:10:30,627Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,633Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,641Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,647Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,654Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,659Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,666Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:10:30,671Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:11:08,714Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [31] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:12:27,050Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [37] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:13:45,386Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [43] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:14:44,844Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:15:03,722Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [49] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:16:22,057Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [55] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:17:40,393Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [61] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:18:58,730Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [67] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:19:44,844Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:20:17,065Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [73] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:20:28,748Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,748Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,762Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,767Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,773Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,778Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,784Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,790Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,797Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:20:28,803Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:21:34,634Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [79] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:22:52,969Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [85] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:24:10,665Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [91] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:24:44,845Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:25:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:25:28,747Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:25:29,002Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [97] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:26:47,338Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [103] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:28:04,138Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [109] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:29:17,865Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [115] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.29.8/10.144.29.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:29:44,845Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:30:10,155Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [1] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_timeout[30s]","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_timeout[30s] | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onTimeout(TcpTransport.java:984) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:30:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,762Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,768Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,777Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,783Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,790Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,795Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,803Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:30:28,809Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:31:04,924Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [121] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:31:33,948Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [7] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:32:22,476Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [127] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:32:51,516Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [13] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:33:40,817Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [133] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:34:09,852Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [19] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:34:44,845Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:34:59,142Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [139] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:35:28,188Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [25] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:35:28,764Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,769Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,775Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,780Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,786Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,790Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,796Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:35:28,800Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:36:17,506Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [145] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:36:46,524Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [31] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:37:35,817Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [151] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:38:04,860Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [37] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:38:54,139Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [157] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:39:23,196Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [43] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:39:44,846Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:40:12,486Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [163] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:40:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:40:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:40:41,532Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [49] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:41:30,822Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [169] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:41:59,868Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [55] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:42:49,156Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [175] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:43:18,204Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [61] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:44:07,493Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [181] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:44:36,540Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [67] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:44:44,846Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:45:25,828Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [187] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:45:54,876Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [73] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:46:44,165Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [193] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:47:13,212Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [79] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:48:01,733Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [199] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:48:30,780Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [85] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:49:20,069Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [205] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:49:44,846Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:49:49,116Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [91] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:50:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:50:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:50:38,405Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [211] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:51:07,452Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [97] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:51:56,730Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [217] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:52:25,788Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [103] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:53:15,076Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [223] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:53:44,124Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [109] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:54:33,410Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [229] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:54:44,847Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:55:02,459Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [115] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:55:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,759Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,764Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,770Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,775Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,780Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,785Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,791Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:28,796Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T19:55:51,749Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [235] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:56:20,796Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [121] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:57:10,083Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [241] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:57:39,132Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [127] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:58:28,422Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [247] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T19:58:57,468Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [133] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T19:59:44,847Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T19:59:46,763Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [253] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:00:15,803Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [139] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:00:28,764Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,768Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,775Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,780Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,784Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,788Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,792Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:00:28,796Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:01:05,094Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [259] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:01:34,139Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [145] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:02:23,428Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [265] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:02:52,476Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [151] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:03:41,765Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [271] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:04:10,812Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [157] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:04:44,848Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:05:00,100Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [277] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:05:28,766Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,771Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,776Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,781Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,787Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,792Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,798Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:28,802Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:05:29,147Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [163] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:06:17,931Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [283] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:06:46,972Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [169] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:07:36,262Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [289] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:08:05,180Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [175] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:08:54,468Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [295] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:09:23,516Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [181] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:09:44,848Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:10:10,507Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [301] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:10:28,744Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:10:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:10:39,548Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [187] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:11:28,835Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [307] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:11:57,883Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [193] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:12:47,171Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [313] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:13:16,220Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [199] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:14:05,509Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [319] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:14:34,556Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [205] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:14:44,849Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:15:23,843Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [325] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:15:29,812Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,813Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,826Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,830Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,836Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,841Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,846Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,850Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,856Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:29,860Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:15:52,892Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [211] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:16:41,547Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [331] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:17:10,588Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [217] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:17:59,875Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [337] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:18:28,925Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [223] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:19:17,571Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [343] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:19:44,849Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:19:46,108Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [229] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:20:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,756Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,759Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,765Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,769Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,774Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,777Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,782Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:28,785Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:20:35,396Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [349] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:21:04,444Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [235] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:21:53,730Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [355] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:22:22,268Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [241] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:23:11,559Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [361] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:23:40,604Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [247] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:24:29,891Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [367] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:24:44,850Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:24:58,940Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [253] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:25:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:25:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:25:48,226Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [373] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:26:17,275Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [259] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:27:06,565Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [379] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:27:35,036Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [265] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:28:24,324Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [385] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:28:53,372Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [271] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:29:42,664Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [391] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:29:44,850Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:30:11,707Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [277] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:30:28,745Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:30:28,746Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1, .opendistro_security], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:31:00,995Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [397] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:31:30,044Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [283] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:32:19,333Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [403] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:32:48,380Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [289] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:33:37,668Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [409] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:34:06,716Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [295] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:34:44,850Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:34:56,005Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [415] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:35:25,052Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [301] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:35:28,805Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,810Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,817Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,821Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,827Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,831Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,836Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:35:28,840Z","level":"DEPRECATION","component":"o.e.d.c.m.IndexNameExpressionResolver","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"this request accesses system indices: [.kibana_task_manager_1], but in a future major version, direct access to system indices will be prevented by default"} | |
| {"timestamp":"2022-02-25T20:36:14,346Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [421] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:36:43,388Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [307] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:37:32,684Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [427] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:38:01,723Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [313] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:38:51,010Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [433] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:39:20,059Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [319] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]","Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300","Caused by: java.net.NoRouteToHostException: No route to host","at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]","at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?]","at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?]","at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]","... 7 more"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception | |
| at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] | |
| at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] | |
| at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?] | |
| at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?] | |
| at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?] | |
| at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.144.51.8/10.144.51.8:9300 | |
| Caused by: java.net.NoRouteToHostException: No route to host | |
| at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] | |
| at sun.nio.ch.Net.pollConnectNow(Net.java:660) ~[?:?] | |
| at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:875) ~[?:?] | |
| at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?] | |
| ... 7 more | |
| {"timestamp":"2022-02-25T20:39:44,851Z","level":"INFO","component":"c.a.o.j.s.JobSweeper","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"Running full sweep"} | |
| {"timestamp":"2022-02-25T20:40:09,348Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-master-us-west1-a-0}{a2Zi_wdITIafm5meTnWxtA}{3dHjyq3HQ8ik4EBByGB-5Q}{10.144.29.8}{10.144.29.8:9300}{m}{region=us-west1, zone=us-west1-a, storageclass=pd-standard} (tried [439] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none}","at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]","at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0]","at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2]","at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]","at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]","at java.lang.Thread.run(Thread.java:832) [?:?]"]} | |
| org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-master-us-west1-a-0][10.144.29.8:9300] handshake failed. unexpected remote node {elasticsearch-logging-coordinator-us-west1-a-0}{sweCmLUHRvWaLr9UjqVnBQ}{b2Fd4Wc0QaygKLxc4ZFBYg}{10.144.29.8}{10.144.29.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} | |
| at org.elasticsearch.transport.TransportService.lambda$connectionValidator$5(TransportService.java:389) ~[elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListener$4.onResponse(ActionListener.java:157) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:476) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.TransportService$5.onResponse(TransportService.java:466) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2] | |
| at com.amazon.opendistroforelasticsearch.security.transport.OpenDistroSecurityInterceptor$RestoringTransportResponseHandler.handleResponse(OpenDistroSecurityInterceptor.java:278) [opendistro_security-1.13.1.0.jar:1.13.1.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.doHandleResponse(InboundHandler.java:253) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.transport.InboundHandler.lambda$handleResponse$1(InboundHandler.java:247) [elasticsearch-7.10.2.jar:7.10.2] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:684) [elasticsearch-7.10.2.jar:7.10.2] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?] | |
| at java.lang.Thread.run(Thread.java:832) [?:?] | |
| {"timestamp":"2022-02-25T20:40:38,395Z","level":"WARN","component":"o.e.c.NodeConnectionsService","cluster.name":"elasticsearch-logging-us-west1","node.name":"elasticsearch-logging-master-us-west1-a-1","message":"failed to connect to {elasticsearch-logging-coordinator-us-west1-a-0}{5WYLVT-tRam6Da249Yu2ag}{k2uDNs2zTIaTpPCbrZuVLg}{10.144.51.8}{10.144.51.8:9300}{region=us-west1, zone=us-west1-a, storageclass=none} (tried [325] times)", "stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [elasticsearch-logging-coordinator-us-west1-a-0][10.144.51.8:9300] connect_exception","at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:978) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$2(ActionListener.java:198) ~[elasticsearch-7.10.2.jar:7.10.2]","at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]","at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]","at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]","at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2152) ~[?:?]","at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.10.2.jar:7.10.2]","at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]","at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]","at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]","at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]","at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]","at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]","at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMa |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment