Opensearch Trace Logs (3 node cluster)

alllogs.log

+ opensearch-cluster-master-0 › opensearch
+ opensearch-cluster-master-1 › opensearch
+ opensearch-cluster-master-2 › opensearch
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:16,305][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] version[1.0.0], pid[9], build[tar/34550c5b17124ddc59458ef774f6b43a086522e3/2021-07-02T23:24:32.510764Z], OS[Linux/4.9.201-tegra-virt-tegra/aarch64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/15.0.1/15.0.1+9]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:16,313][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] JVM home [/usr/share/opensearch/jdk], using bundled JDK [true]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:16,313][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] JVM arguments [-Xshare:auto, -Dopensearch.networkaddress.cache.ttl=60, -Dopensearch.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/opensearch-3459373449892337278, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Dclk.tck=100, -Djdk.attach.allowAttachSelf=true, -Djava.security.policy=/usr/share/opensearch/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy, -Dopensearch.cgroups.hierarchy.override=/, -Xmx512M, -Xms512M, -XX:MaxDirectMemorySize=268435456, -Dopensearch.path.home=/usr/share/opensearch, -Dopensearch.path.conf=/usr/share/opensearch/config, -Dopensearch.distribution.type=tar, -Dopensearch.bundled_jdk=true]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:19,842][INFO ][o.o.s.s.t.SSLConfig      ] [opensearch-cluster-master-2] SSL dual mode is disabled
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:19,843][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] OpenSearch Config path is /usr/share/opensearch/config
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:21,236][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] JVM supports TLSv1.3
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:21,242][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] Config directory is /usr/share/opensearch/config/, from there the key- and truststore files are resolved relatively
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:22,095][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] TLS Transport Client Provider : JDK
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:22,096][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] TLS Transport Server Provider : JDK
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:22,096][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] TLS HTTP Provider             : null
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:22,097][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] Enabled TLS protocols for transport layer : [TLSv1.3, TLSv1.2, TLSv1.1]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:22,097][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-2] Enabled TLS protocols for HTTP layer      : []
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:22,730][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] version[1.0.0], pid[9], build[tar/34550c5b17124ddc59458ef774f6b43a086522e3/2021-07-02T23:24:32.510764Z], OS[Linux/4.9.201-tegra-virt-tegra/aarch64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/15.0.1/15.0.1+9]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:22,742][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] JVM home [/usr/share/opensearch/jdk], using bundled JDK [true]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:22,743][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] JVM arguments [-Xshare:auto, -Dopensearch.networkaddress.cache.ttl=60, -Dopensearch.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/opensearch-13716208375081867309, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Dclk.tck=100, -Djdk.attach.allowAttachSelf=true, -Djava.security.policy=/usr/share/opensearch/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy, -Dopensearch.cgroups.hierarchy.override=/, -Xmx512M, -Xms512M, -XX:MaxDirectMemorySize=268435456, -Dopensearch.path.home=/usr/share/opensearch, -Dopensearch.path.conf=/usr/share/opensearch/config, -Dopensearch.distribution.type=tar, -Dopensearch.bundled_jdk=true]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,011][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] Clustername: opensearch-cluster
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,027][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] Directory /usr/share/opensearch/config has insecure file permissions (should be 0700)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,027][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/opensearch.yml has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,028][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/log4j2.properties has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,028][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/esnode-key.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,029][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/kirk-key.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,029][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/esnode.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,031][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/kirk.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,031][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] File /usr/share/opensearch/config/root-ca.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,466][INFO ][o.o.p.c.PluginSettings   ] [opensearch-cluster-master-2] Config: metricsLocation: /dev/shm/performanceanalyzer/, metricsDeletionInterval: 1, httpsEnabled: false, cleanup-metrics-db-files: true, batch-metrics-retention-period-minutes: 7, rpc-port: 9650, webservice-port 9600
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:23,905][INFO ][o.o.p.PerformanceAnalyzerPlugin] [opensearch-cluster-master-2] Shard IndexingPressure not present in this OpenSearch version. Skipping ShardIndexingPressureMetricsCollector
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:24,918][INFO ][o.o.i.r.ReindexPlugin    ] [opensearch-cluster-master-2] ReindexPlugin reloadSPI called
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:24,924][INFO ][o.o.i.r.ReindexPlugin    ] [opensearch-cluster-master-2] Unable to find any implementation for RemoteReindexExtension
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:24,973][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-2] Loaded scheduler extension: opendistro-index-management, index: .opendistro-ism-config
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,057][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-2] Loaded scheduler extension: reports-scheduler, index: .opendistro-reports-definitions
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,074][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-2] Loaded scheduler extension: opendistro_anomaly_detector, index: .opendistro-anomaly-detector-jobs
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,083][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [aggs-matrix-stats]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,083][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [analysis-common]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,084][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [geo]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,084][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [ingest-common]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,084][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [ingest-geoip]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,084][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [ingest-user-agent]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,089][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [lang-expression]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,091][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [lang-mustache]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,091][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [lang-painless]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,092][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [mapper-extras]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,092][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [opensearch-dashboards]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,092][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [parent-join]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,092][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [percolator]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,092][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [rank-eval]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,093][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [reindex]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,093][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [repository-url]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,093][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded module [transport-netty4]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,095][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-alerting]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,096][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-anomaly-detection]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,096][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-asynchronous-search]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,096][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-index-management]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,097][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-job-scheduler]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,097][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-knn]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,097][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-notebooks]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,097][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-performance-analyzer]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,097][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-reports-scheduler]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,097][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-security]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,098][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-2] loaded plugin [opensearch-sql]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,242][INFO ][o.o.e.NodeEnvironment    ] [opensearch-cluster-master-2] using [1] data paths, mounts [[/usr/share/opensearch/data (/dev/longhorn/pvc-d9a5b7ea-ece4-4a7a-96e1-b450ba153f84)]], net usable_space [7.7gb], net total_space [7.8gb], types [ext4]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,243][INFO ][o.o.e.NodeEnvironment    ] [opensearch-cluster-master-2] heap size [512mb], compressed ordinary object pointers [true]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,258][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,268][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,271][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,271][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,273][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,398][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,400][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,401][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,401][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:25,405][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] node name [opensearch-cluster-master-2], node ID [61sMC4gJQCiPVS0y-UMGJw], cluster name [opensearch-cluster], roles [master, remote_cluster_client, data, ingest]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:28,812][INFO ][o.o.s.s.t.SSLConfig      ] [opensearch-cluster-master-0] SSL dual mode is disabled
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:28,812][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] OpenSearch Config path is /usr/share/opensearch/config
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:30,263][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] JVM supports TLSv1.3
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:30,276][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] Config directory is /usr/share/opensearch/config/, from there the key- and truststore files are resolved relatively
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:31,479][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] TLS Transport Client Provider : JDK
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:31,480][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] TLS Transport Server Provider : JDK
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:31,480][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] TLS HTTP Provider             : null
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:31,480][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] Enabled TLS protocols for transport layer : [TLSv1.3, TLSv1.2, TLSv1.1]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:31,480][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-0] Enabled TLS protocols for HTTP layer      : []
opensearch-cluster-master-1 opensearch [2021-09-22T03:07:31,616][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] version[1.0.0], pid[10], build[tar/34550c5b17124ddc59458ef774f6b43a086522e3/2021-07-02T23:24:32.510764Z], OS[Linux/5.10.60-v8+/aarch64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/15.0.1/15.0.1+9]
opensearch-cluster-master-1 opensearch [2021-09-22T03:07:31,633][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] JVM home [/usr/share/opensearch/jdk], using bundled JDK [true]
opensearch-cluster-master-1 opensearch [2021-09-22T03:07:31,634][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] JVM arguments [-Xshare:auto, -Dopensearch.networkaddress.cache.ttl=60, -Dopensearch.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/opensearch-10491649970703885794, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Dclk.tck=100, -Djdk.attach.allowAttachSelf=true, -Djava.security.policy=/usr/share/opensearch/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy, -Dopensearch.cgroups.hierarchy.override=/, -Xmx512M, -Xms512M, -XX:MaxDirectMemorySize=268435456, -Dopensearch.path.home=/usr/share/opensearch, -Dopensearch.path.conf=/usr/share/opensearch/config, -Dopensearch.distribution.type=tar, -Dopensearch.bundled_jdk=true]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:32,330][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [discovery.zen.minimum_master_nodes] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:32,332][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:32,333][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:32,333][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:32,335][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,785][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] Clustername: opensearch-cluster
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,813][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] Directory /usr/share/opensearch/config has insecure file permissions (should be 0700)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,813][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/log4j2.properties has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,814][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/opensearch.yml has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,815][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/esnode.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,816][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/kirk-key.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,818][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/esnode-key.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,820][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/root-ca.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:32,821][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] File /usr/share/opensearch/config/kirk.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:33,168][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:33,249][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:33,335][INFO ][o.o.p.c.PluginSettings   ] [opensearch-cluster-master-0] Config: metricsLocation: /dev/shm/performanceanalyzer/, metricsDeletionInterval: 1, httpsEnabled: false, cleanup-metrics-db-files: true, batch-metrics-retention-period-minutes: 7, rpc-port: 9650, webservice-port 9600
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:33,757][INFO ][o.o.p.PerformanceAnalyzerPlugin] [opensearch-cluster-master-0] Shard IndexingPressure not present in this OpenSearch version. Skipping ShardIndexingPressureMetricsCollector
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,123][INFO ][o.o.i.r.ReindexPlugin    ] [opensearch-cluster-master-0] ReindexPlugin reloadSPI called
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,127][INFO ][o.o.i.r.ReindexPlugin    ] [opensearch-cluster-master-0] Unable to find any implementation for RemoteReindexExtension
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,183][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-0] Loaded scheduler extension: opendistro-index-management, index: .opendistro-ism-config
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,201][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-0] Loaded scheduler extension: opendistro_anomaly_detector, index: .opendistro-anomaly-detector-jobs
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,276][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-0] Loaded scheduler extension: reports-scheduler, index: .opendistro-reports-definitions
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,284][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [aggs-matrix-stats]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,285][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [analysis-common]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,285][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [geo]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,285][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [ingest-common]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:35,294][WARN ][o.o.s.c.Salt             ] [opensearch-cluster-master-2] If you plan to use field masking pls configure compliance salt e1ukloTsQlOgPquJ to be a random string of 16 chars length identical on all nodes
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,285][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [ingest-geoip]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,291][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [ingest-user-agent]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,292][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [lang-expression]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,292][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [lang-mustache]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,292][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [lang-painless]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,292][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [mapper-extras]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,293][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [opensearch-dashboards]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,293][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [parent-join]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,293][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [percolator]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,293][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [rank-eval]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,293][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [reindex]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,294][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [repository-url]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,294][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded module [transport-netty4]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:35,352][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Message routing enabled: true
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,296][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-alerting]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,297][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-anomaly-detection]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,297][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-asynchronous-search]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,297][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-index-management]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,298][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-job-scheduler]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,298][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-knn]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,298][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-notebooks]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,298][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-performance-analyzer]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,298][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-reports-scheduler]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,299][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-security]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:35,299][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-0] loaded plugin [opensearch-sql]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:35,497][INFO ][o.o.s.f.SecurityFilter   ] [opensearch-cluster-master-2] <NONE> indices are made immutable.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:36,363][INFO ][o.o.a.b.ADCircuitBreakerService] [opensearch-cluster-master-2] Registered memory breaker.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,336][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,341][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,387][INFO ][o.o.t.NettyAllocator     ] [opensearch-cluster-master-2] creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=256kb, factors={opensearch.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=1mb, heap_size=512mb}]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,404][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,405][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,406][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,407][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,407][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,424][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,428][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,476][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,476][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,477][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,478][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,479][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,479][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,480][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,480][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,493][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,501][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,501][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,502][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,504][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,519][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,597][INFO ][o.o.d.DiscoveryModule    ] [opensearch-cluster-master-2] using discovery type [zen] and seed hosts providers [settings]
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:37,627][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:38,558][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:38,568][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:38,569][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:38,570][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:38,573][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:38,623][WARN ][o.o.g.DanglingIndicesState] [opensearch-cluster-master-2] gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,194][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,595][INFO ][o.o.p.h.c.PerformanceAnalyzerConfigAction] [opensearch-cluster-master-2] PerformanceAnalyzer Enabled: false
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,723][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] initialized
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,724][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] starting ...
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,739][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,740][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,740][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,741][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,749][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,974][INFO ][o.o.t.TransportService   ] [opensearch-cluster-master-2] publish_address {10.42.8.149:9300}, bound_addresses {[::]:9300}
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,976][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,977][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,977][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,978][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,984][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,985][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,986][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,987][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:39,988][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:07:40,016][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:07:41,819][INFO ][o.o.s.s.t.SSLConfig      ] [opensearch-cluster-master-1] SSL dual mode is disabled
opensearch-cluster-master-1 opensearch [2021-09-22T03:07:41,820][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] OpenSearch Config path is /usr/share/opensearch/config
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,095][INFO ][o.o.e.NodeEnvironment    ] [opensearch-cluster-master-0] using [1] data paths, mounts [[/usr/share/opensearch/data (/dev/longhorn/pvc-e14ab63e-b2d5-4696-a2f3-7f4f6302d6ca)]], net usable_space [7.7gb], net total_space [7.8gb], types [ext4]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,096][INFO ][o.o.e.NodeEnvironment    ] [opensearch-cluster-master-0] heap size [512mb], compressed ordinary object pointers [true]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,126][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,145][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,146][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,147][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,147][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,280][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,282][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,282][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,283][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:45,288][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] node name [opensearch-cluster-master-0], node ID [PRO48_2LToSUQX8dSnZZnA], cluster name [opensearch-cluster], roles [master, remote_cluster_client, data, ingest]
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:53,800][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [discovery.zen.minimum_master_nodes] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:53,802][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:53,802][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:53,803][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:53,815][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:54,757][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:54,853][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:58,610][WARN ][o.o.s.c.Salt             ] [opensearch-cluster-master-0] If you plan to use field masking pls configure compliance salt e1ukloTsQlOgPquJ to be a random string of 16 chars length identical on all nodes
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:58,677][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Message routing enabled: true
opensearch-cluster-master-0 opensearch [2021-09-22T03:07:58,846][INFO ][o.o.s.f.SecurityFilter   ] [opensearch-cluster-master-0] <NONE> indices are made immutable.
opensearch-cluster-master-1 opensearch [2021-09-22T03:07:59,559][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] JVM supports TLSv1.3
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:00,061][INFO ][o.o.a.b.ADCircuitBreakerService] [opensearch-cluster-master-0] Registered memory breaker.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:01,485][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:01,486][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:01,486][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:01,487][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:01,579][INFO ][o.o.b.BootstrapChecks    ] [opensearch-cluster-master-2] bound or publishing to a non-loopback address, enforcing bootstrap checks
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:01,944][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:01,950][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,055][INFO ][o.o.t.NettyAllocator     ] [opensearch-cluster-master-0] creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=256kb, factors={opensearch.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=1mb, heap_size=512mb}]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,093][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,106][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,107][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,107][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,108][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,134][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,136][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,195][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,196][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,196][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,197][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,197][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,198][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,198][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,198][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,222][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,229][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,230][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,231][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,231][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,261][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,398][INFO ][o.o.d.DiscoveryModule    ] [opensearch-cluster-master-0] using discovery type [zen] and seed hosts providers [settings]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:02,430][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:03,044][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] Config directory is /usr/share/opensearch/config/, from there the key- and truststore files are resolved relatively
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:03,792][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:03,802][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:03,803][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:03,806][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:03,811][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:03,850][WARN ][o.o.g.DanglingIndicesState] [opensearch-cluster-master-0] gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:04,512][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,090][INFO ][o.o.p.h.c.PerformanceAnalyzerConfigAction] [opensearch-cluster-master-0] PerformanceAnalyzer Enabled: false
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,248][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] initialized
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,248][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] starting ...
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,268][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,269][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,269][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,270][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,277][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,595][INFO ][o.o.t.TransportService   ] [opensearch-cluster-master-0] publish_address {10.42.0.146:9300}, bound_addresses {[::]:9300}
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,596][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,596][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,598][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,598][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,604][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,605][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,605][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,606][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,606][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:05,644][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:06,403][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:06,403][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:06,404][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:06,404][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:06,698][INFO ][o.o.b.BootstrapChecks    ] [opensearch-cluster-master-0] bound or publishing to a non-loopback address, enforcing bootstrap checks
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:08,741][INFO ][o.o.c.c.Coordinator      ] [opensearch-cluster-master-0] setting initial configuration to VotingConfiguration{PRO48_2LToSUQX8dSnZZnA,{bootstrap-placeholder}-opensearch-cluster-master-1,61sMC4gJQCiPVS0y-UMGJw}
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:09,374][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] TLS Transport Client Provider : JDK
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:09,374][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] TLS Transport Server Provider : JDK
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:09,375][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] TLS HTTP Provider             : null
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:09,376][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] Enabled TLS protocols for transport layer : [TLSv1.3, TLSv1.2, TLSv1.1]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:09,376][INFO ][o.o.s.s.DefaultSecurityKeyStore] [opensearch-cluster-master-1] Enabled TLS protocols for HTTP layer      : []
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:09,892][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-2] failed to join {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, targetNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}]}
opensearch-cluster-master-2 opensearch org.opensearch.transport.NodeNotConnectedException: [opensearch-cluster-master-0][10.42.0.146:9300] Node not connected
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:202) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService.getConnection(TransportService.java:706) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService.sendRequest(TransportService.java:626) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:317) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:242) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$3(JoinHelper.java:166) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:314) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:318) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:09,919][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-2] failed to join {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, targetNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}]}
opensearch-cluster-master-2 opensearch org.opensearch.transport.NodeNotConnectedException: [opensearch-cluster-master-0][10.42.0.146:9300] Node not connected
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:202) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService.getConnection(TransportService.java:706) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService.sendRequest(TransportService.java:626) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:317) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:242) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$3(JoinHelper.java:166) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:314) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:318) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:09,939][INFO ][o.o.c.c.Coordinator      ] [opensearch-cluster-master-2] setting initial configuration to VotingConfiguration{PRO48_2LToSUQX8dSnZZnA,{bootstrap-placeholder}-opensearch-cluster-master-1,61sMC4gJQCiPVS0y-UMGJw}
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:10,328][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-0] failed to join {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, targetNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}]}
opensearch-cluster-master-0 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-0][10.42.0.146:9300][internal:cluster/coordination/join]
opensearch-cluster-master-0 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:10,364][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-0] [discovery.zen.minimum_master_nodes] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,479][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] Clustername: opensearch-cluster
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,502][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] Directory /usr/share/opensearch/config has insecure file permissions (should be 0700)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,503][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/log4j2.properties has insecure file permissions (should be 0600)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,505][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/opensearch.yml has insecure file permissions (should be 0600)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,506][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/kirk.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,507][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/root-ca.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,508][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/esnode-key.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,509][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/esnode.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:10,510][WARN ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] File /usr/share/opensearch/config/kirk-key.pem has insecure file permissions (should be 0600)
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:10,572][INFO ][o.o.c.s.MasterService    ] [opensearch-cluster-master-0] elected-as-master ([2] nodes joined)[{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} elect leader, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 2, version: 1, delta: master node changed {previous [], current [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}]}, added {{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}}
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:10,593][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-2] [discovery.zen.minimum_master_nodes] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:10,578][WARN ][o.o.c.s.MasterService    ] [opensearch-cluster-master-0] failing [elected-as-master ([2] nodes joined)[{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} elect leader, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_]]: failed to commit cluster state version [1]
opensearch-cluster-master-0 opensearch org.opensearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 2 while handling publication
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.Coordinator.publish(Coordinator.java:1097) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService.publish(MasterService.java:281) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService.runTasks(MasterService.java:263) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService.access$000(MasterService.java:86) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService$Batcher.run(MasterService.java:164) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:163) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:201) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:697) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedOpenSearchThreadPoolExecutor.java:266) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedOpenSearchThreadPoolExecutor.java:229) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:10,583][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-0] failed to join {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, minimumTerm=1, optionalJoin=Optional[Join{term=2, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, targetNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}]}
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:10,655][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-2] failed to join {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, minimumTerm=1, optionalJoin=Optional[Join{term=2, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, targetNode={opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}]}
opensearch-cluster-master-2 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-0][10.42.0.146:9300][internal:cluster/coordination/join]
opensearch-cluster-master-0 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-0][10.42.0.146:9300][internal:cluster/coordination/join]
opensearch-cluster-master-0 opensearch Caused by: org.opensearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 2 while handling publication
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.Coordinator.publish(Coordinator.java:1097) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService.publish(MasterService.java:281) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService.runTasks(MasterService.java:263) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService.access$000(MasterService.java:86) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.MasterService$Batcher.run(MasterService.java:164) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:163) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:201) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:697) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedOpenSearchThreadPoolExecutor.java:266) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedOpenSearchThreadPoolExecutor.java:229) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch Caused by: org.opensearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 2 while handling publication
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator.publish(Coordinator.java:1097) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.service.MasterService.publish(MasterService.java:281) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.service.MasterService.runTasks(MasterService.java:263) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.service.MasterService.access$000(MasterService.java:86) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.service.MasterService$Batcher.run(MasterService.java:164) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:163) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:201) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:697) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedOpenSearchThreadPoolExecutor.java:266) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedOpenSearchThreadPoolExecutor.java:229) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:10,719][INFO ][o.o.c.s.MasterService    ] [opensearch-cluster-master-2] elected-as-master ([2] nodes joined)[{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} elect leader, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 3, version: 1, delta: master node changed {previous [], current [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}]}, added {{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:11,192][INFO ][o.o.p.c.PluginSettings   ] [opensearch-cluster-master-1] Config: metricsLocation: /dev/shm/performanceanalyzer/, metricsDeletionInterval: 1, httpsEnabled: false, cleanup-metrics-db-files: true, batch-metrics-retention-period-minutes: 7, rpc-port: 9650, webservice-port 9600
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:20,758][INFO ][o.o.c.c.C.CoordinatorPublication] [opensearch-cluster-master-2] after [10s] publication of cluster state version [1] is still waiting for {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} [SENT_PUBLISH_REQUEST], {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr} [WAITING_FOR_QUORUM]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:28,108][INFO ][o.o.p.PerformanceAnalyzerPlugin] [opensearch-cluster-master-1] Shard IndexingPressure not present in this OpenSearch version. Skipping ShardIndexingPressureMetricsCollector
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,186][INFO ][o.o.i.r.ReindexPlugin    ] [opensearch-cluster-master-1] ReindexPlugin reloadSPI called
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,189][INFO ][o.o.i.r.ReindexPlugin    ] [opensearch-cluster-master-1] Unable to find any implementation for RemoteReindexExtension
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,296][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-1] Loaded scheduler extension: opendistro-index-management, index: .opendistro-ism-config
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,471][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-1] Loaded scheduler extension: reports-scheduler, index: .opendistro-reports-definitions
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,506][INFO ][o.o.j.JobSchedulerPlugin ] [opensearch-cluster-master-1] Loaded scheduler extension: opendistro_anomaly_detector, index: .opendistro-anomaly-detector-jobs
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,521][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [aggs-matrix-stats]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,522][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [analysis-common]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,522][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [geo]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,523][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [ingest-common]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,523][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [ingest-geoip]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,524][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [ingest-user-agent]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,524][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [lang-expression]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,525][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [lang-mustache]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,525][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [lang-painless]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,526][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [mapper-extras]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,526][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [opensearch-dashboards]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,527][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [parent-join]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,527][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [percolator]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,528][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [rank-eval]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,529][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [reindex]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,531][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [repository-url]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,532][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded module [transport-netty4]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,545][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-alerting]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,546][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-anomaly-detection]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,546][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-asynchronous-search]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,547][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-index-management]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,548][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-job-scheduler]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,548][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-knn]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,549][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-notebooks]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,550][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-performance-analyzer]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,550][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-reports-scheduler]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,551][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-security]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:30,551][INFO ][o.o.p.PluginsService     ] [opensearch-cluster-master-1] loaded plugin [opensearch-sql]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,073][INFO ][o.o.e.NodeEnvironment    ] [opensearch-cluster-master-1] using [1] data paths, mounts [[/usr/share/opensearch/data (/dev/longhorn/pvc-1cf0c43a-1d44-4459-a1ad-6223b7ac8547)]], net usable_space [7.7gb], net total_space [7.8gb], types [ext4]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,077][INFO ][o.o.e.NodeEnvironment    ] [opensearch-cluster-master-1] heap size [512mb], compressed ordinary object pointers [true]
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,121][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,140][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,143][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,146][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,149][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,353][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,355][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,360][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,362][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:31,383][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] node name [opensearch-cluster-master-1], node ID [pTorOyp-S6uhV4GkS1dklA], cluster name [opensearch-cluster], roles [master, remote_cluster_client, data, ingest]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,413][WARN ][o.o.g.PersistedClusterStateService] [opensearch-cluster-master-0] writing cluster state took [20539ms] which is above the warn threshold of [10s]; wrote full state with [0] indices
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,438][INFO ][o.o.c.c.CoordinationState] [opensearch-cluster-master-0] cluster UUID set to [sl5LbeGzSDyZr_9lzSTRUw]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,434][INFO ][o.o.c.c.CoordinationState] [opensearch-cluster-master-2] cluster UUID set to [sl5LbeGzSDyZr_9lzSTRUw]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,659][WARN ][o.o.n.Node               ] [opensearch-cluster-master-2] timed out while waiting for initial discovery state - timeout: 30s
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,740][INFO ][o.o.h.AbstractHttpServerTransport] [opensearch-cluster-master-2] publish_address {10.42.8.149:9200}, bound_addresses {[::]:9200}
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,740][INFO ][o.o.n.Node               ] [opensearch-cluster-master-2] started
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,743][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] Node started
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,744][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-2] Will attempt to create index .opendistro_security and default configs if they are absent
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,752][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-2] Background init thread started. Install default config?: true
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,754][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-2] 0 OpenSearch Security modules loaded so far: []
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,760][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-0] master node changed {previous [], current [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}]}, added {{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}}, term: 3, version: 1, reason: ApplyCommitRequest{term=3, version=1, sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}}
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,852][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-0] CLuster is not recovered yet.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:31,894][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-2] master node changed {previous [], current [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}]}, added {{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}, term: 3, version: 1, reason: Publication{term=3, version=1}
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,897][INFO ][o.o.h.AbstractHttpServerTransport] [opensearch-cluster-master-0] publish_address {10.42.0.146:9200}, bound_addresses {[::]:9200}
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,900][INFO ][o.o.n.Node               ] [opensearch-cluster-master-0] started
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,903][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] Node started
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,903][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Will attempt to create index .opendistro_security and default configs if they are absent
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,904][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Background init thread started. Install default config?: true
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:31,906][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-0] 0 OpenSearch Security modules loaded so far: []
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:32,012][WARN ][o.o.p.c.s.h.ConfigOverridesClusterSettingHandler] [opensearch-cluster-master-2] Config override setting update called with empty string. Ignoring.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:32,020][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-2] CLuster is not recovered yet.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:32,027][INFO ][o.o.c.r.a.DiskThresholdMonitor] [opensearch-cluster-master-2] skipping monitor as a check is already in progress
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:32,065][WARN ][o.o.p.c.s.h.ConfigOverridesClusterSettingHandler] [opensearch-cluster-master-0] Config override setting update called with empty string. Ignoring.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:32,288][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-0] CLuster is not recovered yet.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:32,376][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-2] CLuster is not recovered yet.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:32,660][INFO ][o.o.g.GatewayService     ] [opensearch-cluster-master-2] recovered [0] indices into cluster_state
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:33,180][INFO ][o.o.c.m.MetadataCreateIndexService] [opensearch-cluster-master-2] [.opendistro_security] creating index, cause [api], templates [], shards [1]/[1]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:34,356][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Index .opendistro_security already exists
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:34,357][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Node started, try to initialize it. Wait for at least yellow cluster state....
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:36,166][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-2] Index .opendistro_security created?: true
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:36,167][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-2] Node started, try to initialize it. Wait for at least yellow cluster state....
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:36,189][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'config' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:36,180][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'config' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:37,084][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] create_mapping [_doc]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:37,878][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Index .opendistro_security already contains doc with id config, skipping update.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:37,879][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'roles' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:37,983][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Doc with id 'config' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:37,983][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'roles' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:38,107][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:38,996][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Index .opendistro_security already contains doc with id roles, skipping update.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:38,994][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Doc with id 'roles' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:38,997][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'rolesmapping' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:38,995][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'rolesmapping' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:39,103][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:39,689][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Index .opendistro_security already contains doc with id rolesmapping, skipping update.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:39,690][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'internalusers' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:39,727][INFO ][o.o.c.r.a.AllocationService] [opensearch-cluster-master-2] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.opendistro_security][0]]]).
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:39,769][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Doc with id 'rolesmapping' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:39,770][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'internalusers' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:40,031][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:40,848][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Index .opendistro_security already contains doc with id internalusers, skipping update.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:40,848][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'actiongroups' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:40,951][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Doc with id 'internalusers' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:40,952][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'actiongroups' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:41,000][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:41,665][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Index .opendistro_security already contains doc with id actiongroups, skipping update.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:41,665][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'tenants' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:41,739][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:41,750][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Doc with id 'actiongroups' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:41,750][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'tenants' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:42,273][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Index .opendistro_security already contains doc with id tenants, skipping update.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:42,274][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'nodesdn' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=true
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:42,343][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:42,385][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Doc with id 'tenants' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:42,385][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'nodesdn' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=true
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:42,903][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Index .opendistro_security already contains doc with id nodesdn, skipping update.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:42,907][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'whitelist' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/whitelist.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=true
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:42,974][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Doc with id 'nodesdn' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:42,976][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'whitelist' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/whitelist.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=true
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:43,009][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:43,676][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Index .opendistro_security already contains doc with id whitelist, skipping update.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:43,677][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Will update 'audit' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/audit.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:43,802][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Doc with id 'whitelist' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:43,802][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Will update 'audit' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/audit.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:43,883][INFO ][o.o.c.m.MetadataMappingService] [opensearch-cluster-master-2] [.opendistro_security/oqAdL1F-TbeZVzCwElOr9Q] update_mapping [_doc]
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:44,402][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-0] Index .opendistro_security already contains doc with id audit, skipping update.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,476][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-2] Doc with id 'audit' and version 2 is updated in .opendistro_security index.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,974][INFO ][stdout                   ] [opensearch-cluster-master-2] [FINE] No subscribers registered for event class org.opensearch.security.securityconf.DynamicConfigFactory$NodesDnModelImpl
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,978][INFO ][stdout                   ] [opensearch-cluster-master-2] [FINE] No subscribers registered for event class org.greenrobot.eventbus.NoSubscriberEvent
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,979][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing on REST API is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,979][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] [AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from REST API auditing.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,982][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing on Transport API is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,983][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] [AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from Transport API auditing.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,983][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing of request body is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,983][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Bulk requests resolution is disabled during request auditing.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,984][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Index resolution is enabled during request auditing.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,984][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Sensitive headers auditing is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,984][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing requests from kibanaserver users is disabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,992][WARN ][o.o.s.a.r.AuditMessageRouter] [opensearch-cluster-master-2] No endpoint configured for categories [BAD_HEADERS, FAILED_LOGIN, MISSING_PRIVILEGES, GRANTED_PRIVILEGES, OPENDISTRO_SECURITY_INDEX_ATTEMPT, SSL_EXCEPTION, AUTHENTICATED, INDEX_EVENT, COMPLIANCE_DOC_READ, COMPLIANCE_DOC_WRITE, COMPLIANCE_EXTERNAL_CONFIG, COMPLIANCE_INTERNAL_CONFIG_READ, COMPLIANCE_INTERNAL_CONFIG_WRITE], using default endpoint
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,993][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing of external configuration is disabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,993][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing of internal configuration is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,996][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing only metadata information for read request is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,996][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing will watch {} for read requests.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing read operation requests from kibanaserver users is disabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing only metadata information for write request is enabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing diffs for write requests is disabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing write operation requests from kibanaserver users is disabled.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Auditing will watch <NONE> for write requests.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] .opendistro_security is used as internal security index.
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,997][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-2] Internal index used for posting audit logs is null
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:44,999][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-2] Hot-reloading of audit configuration is enabled
opensearch-cluster-master-2 opensearch [2021-09-22T03:08:45,000][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-2] Node 'opensearch-cluster-master-2' initialized
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,305][INFO ][stdout                   ] [opensearch-cluster-master-0] [FINE] No subscribers registered for event class org.opensearch.security.securityconf.DynamicConfigFactory$NodesDnModelImpl
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,306][INFO ][stdout                   ] [opensearch-cluster-master-0] [FINE] No subscribers registered for event class org.greenrobot.eventbus.NoSubscriberEvent
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,307][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing on REST API is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,307][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] [AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from REST API auditing.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,307][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing on Transport API is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,307][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] [AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from Transport API auditing.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,308][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing of request body is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,308][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Bulk requests resolution is disabled during request auditing.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,308][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Index resolution is enabled during request auditing.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,308][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Sensitive headers auditing is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,308][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing requests from kibanaserver users is disabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,344][WARN ][o.o.s.a.r.AuditMessageRouter] [opensearch-cluster-master-0] No endpoint configured for categories [BAD_HEADERS, FAILED_LOGIN, MISSING_PRIVILEGES, GRANTED_PRIVILEGES, OPENDISTRO_SECURITY_INDEX_ATTEMPT, SSL_EXCEPTION, AUTHENTICATED, INDEX_EVENT, COMPLIANCE_DOC_READ, COMPLIANCE_DOC_WRITE, COMPLIANCE_EXTERNAL_CONFIG, COMPLIANCE_INTERNAL_CONFIG_READ, COMPLIANCE_INTERNAL_CONFIG_WRITE], using default endpoint
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,345][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing of external configuration is disabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,345][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing of internal configuration is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,345][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing only metadata information for read request is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,349][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing will watch {} for read requests.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,349][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing read operation requests from kibanaserver users is disabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,387][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing only metadata information for write request is enabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,387][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing diffs for write requests is disabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,387][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing write operation requests from kibanaserver users is disabled.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,387][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Auditing will watch <NONE> for write requests.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,387][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] .opendistro_security is used as internal security index.
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,388][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-0] Internal index used for posting audit logs is null
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,389][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Hot-reloading of audit configuration is enabled
opensearch-cluster-master-0 opensearch [2021-09-22T03:08:45,390][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-0] Node 'opensearch-cluster-master-0' initialized
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:52,926][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [discovery.zen.minimum_master_nodes] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:52,929][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:52,930][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:52,934][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:52,937][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:54,507][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:54,647][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:58,573][WARN ][o.o.s.c.Salt             ] [opensearch-cluster-master-1] If you plan to use field masking pls configure compliance salt e1ukloTsQlOgPquJ to be a random string of 16 chars length identical on all nodes
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:58,654][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Message routing enabled: true
opensearch-cluster-master-1 opensearch [2021-09-22T03:08:58,890][INFO ][o.o.s.f.SecurityFilter   ] [opensearch-cluster-master-1] <NONE> indices are made immutable.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:00,095][INFO ][o.o.a.b.ADCircuitBreakerService] [opensearch-cluster-master-1] Registered memory breaker.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,727][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,734][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,822][INFO ][o.o.t.NettyAllocator     ] [opensearch-cluster-master-1] creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=256kb, factors={opensearch.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=1mb, heap_size=512mb}]
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,854][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,855][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,857][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,858][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,859][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,895][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:01,898][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,033][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,034][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,036][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,037][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,038][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,039][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,040][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,042][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,096][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,115][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,116][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,118][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,119][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,141][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,356][INFO ][o.o.d.DiscoveryModule    ] [opensearch-cluster-master-1] using discovery type [zen] and seed hosts providers [settings]
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:02,410][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,171][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,182][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,184][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,188][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,190][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,242][WARN ][o.o.g.DanglingIndicesState] [opensearch-cluster-master-1] gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:04,983][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,616][INFO ][o.o.p.h.c.PerformanceAnalyzerConfigAction] [opensearch-cluster-master-1] PerformanceAnalyzer Enabled: false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,892][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] initialized
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,893][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] starting ...
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,940][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,941][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,942][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,943][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:05,956][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,720][INFO ][o.o.t.TransportService   ] [opensearch-cluster-master-1] publish_address {10.42.12.53:9300}, bound_addresses {[::]:9300}
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,722][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,728][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,730][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,731][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,742][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,743][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,747][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,751][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,757][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:06,820][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:07,558][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.data] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:07,561][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.remote_cluster_client] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:07,562][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.ingest] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:07,563][DEPRECATION][o.o.d.c.s.Settings       ] [opensearch-cluster-master-1] [node.master] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:07,699][INFO ][o.o.b.BootstrapChecks    ] [opensearch-cluster-master-1] bound or publishing to a non-loopback address, enforcing bootstrap checks
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:11,040][INFO ][o.o.c.c.Coordinator      ] [opensearch-cluster-master-1] setting initial configuration to VotingConfiguration{PRO48_2LToSUQX8dSnZZnA,pTorOyp-S6uhV4GkS1dklA,61sMC4gJQCiPVS0y-UMGJw}
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:12,868][INFO ][o.o.c.r.a.AllocationService] [opensearch-cluster-master-2] updating number_of_replicas to [2] for indices [.opendistro_security]
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:12,870][INFO ][o.o.c.s.MasterService    ] [opensearch-cluster-master-2] node-join[{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} join existing leader], term: 3, version: 17, delta: added {{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}
opensearch-cluster-master-0 opensearch [2021-09-22T03:09:13,482][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-0] added {{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}, term: 3, version: 17, reason: ApplyCommitRequest{term=3, version=17, sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}}
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:13,756][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-1] master node changed {previous [], current [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}]}, added {{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr},{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}}, term: 3, version: 17, reason: ApplyCommitRequest{term=3, version=17, sourceNode={opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}}
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:13,843][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-1] Data node added  PRO48_2LToSUQX8dSnZZnA
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:13,843][INFO ][o.o.a.c.HashRing         ] [opensearch-cluster-master-1] Rebuild hash ring
opensearch-cluster-master-0 opensearch [2021-09-22T03:09:13,948][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-0] Data node added  pTorOyp-S6uhV4GkS1dklA
opensearch-cluster-master-0 opensearch [2021-09-22T03:09:13,948][INFO ][o.o.a.c.HashRing         ] [opensearch-cluster-master-0] Rebuild hash ring
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:13,962][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-2] added {{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}, term: 3, version: 17, reason: Publication{term=3, version=17}
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:13,989][INFO ][o.o.a.c.ADClusterEventListener] [opensearch-cluster-master-2] Data node added  pTorOyp-S6uhV4GkS1dklA
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:13,990][INFO ][o.o.a.c.HashRing         ] [opensearch-cluster-master-2] Rebuild hash ring
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,011][INFO ][o.o.h.AbstractHttpServerTransport] [opensearch-cluster-master-1] publish_address {10.42.12.53:9200}, bound_addresses {[::]:9200}
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,011][INFO ][o.o.n.Node               ] [opensearch-cluster-master-1] started
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,014][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] Node started
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,015][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Will attempt to create index .opendistro_security and default configs if they are absent
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,020][INFO ][o.o.s.OpenSearchSecurityPlugin] [opensearch-cluster-master-1] 0 OpenSearch Security modules loaded so far: []
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,023][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Background init thread started. Install default config?: true
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:14,081][WARN ][o.o.p.c.s.h.ConfigOverridesClusterSettingHandler] [opensearch-cluster-master-1] Config override setting update called with empty string. Ignoring.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,186][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Index .opendistro_security already exists
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,187][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Node started, try to initialize it. Wait for at least yellow cluster state....
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,237][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'config' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,562][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id config, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,563][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'roles' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,669][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id roles, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,671][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'rolesmapping' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,787][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id rolesmapping, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,789][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'internalusers' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,861][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id internalusers, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,862][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'actiongroups' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,933][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id actiongroups, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:16,933][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'tenants' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,007][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id tenants, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,010][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'nodesdn' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=true
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,071][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id nodesdn, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,075][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'whitelist' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/whitelist.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=true
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,129][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id whitelist, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,138][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Will update 'audit' with /usr/share/opensearch/plugins/opensearch-security/securityconfig/audit.yml and populate it with empty doc if file missing and populateEmptyIfFileMissing=false
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:17,456][INFO ][o.o.s.s.ConfigHelper     ] [opensearch-cluster-master-1] Index .opendistro_security already contains doc with id audit, skipping update.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,404][INFO ][stdout                   ] [opensearch-cluster-master-1] [FINE] No subscribers registered for event class org.opensearch.security.securityconf.DynamicConfigFactory$NodesDnModelImpl
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,408][INFO ][stdout                   ] [opensearch-cluster-master-1] [FINE] No subscribers registered for event class org.greenrobot.eventbus.NoSubscriberEvent
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,409][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing on REST API is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,410][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] [AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from REST API auditing.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,410][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing on Transport API is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,411][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] [AUTHENTICATED, GRANTED_PRIVILEGES] are excluded from Transport API auditing.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,411][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing of request body is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,412][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Bulk requests resolution is disabled during request auditing.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,412][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Index resolution is enabled during request auditing.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,413][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Sensitive headers auditing is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,413][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing requests from kibanaserver users is disabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,430][WARN ][o.o.s.a.r.AuditMessageRouter] [opensearch-cluster-master-1] No endpoint configured for categories [BAD_HEADERS, FAILED_LOGIN, MISSING_PRIVILEGES, GRANTED_PRIVILEGES, OPENDISTRO_SECURITY_INDEX_ATTEMPT, SSL_EXCEPTION, AUTHENTICATED, INDEX_EVENT, COMPLIANCE_DOC_READ, COMPLIANCE_DOC_WRITE, COMPLIANCE_EXTERNAL_CONFIG, COMPLIANCE_INTERNAL_CONFIG_READ, COMPLIANCE_INTERNAL_CONFIG_WRITE], using default endpoint
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,431][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing of external configuration is disabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,432][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing of internal configuration is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,432][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing only metadata information for read request is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,433][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing will watch {} for read requests.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,434][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing read operation requests from kibanaserver users is disabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,434][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing only metadata information for write request is enabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,435][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing diffs for write requests is disabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,435][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing write operation requests from kibanaserver users is disabled.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,436][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Auditing will watch <NONE> for write requests.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,436][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] .opendistro_security is used as internal security index.
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,436][INFO ][o.o.s.a.i.AuditLogImpl   ] [opensearch-cluster-master-1] Internal index used for posting audit logs is null
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,442][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Hot-reloading of audit configuration is enabled
opensearch-cluster-master-1 opensearch [2021-09-22T03:09:18,442][INFO ][o.o.s.c.ConfigurationRepository] [opensearch-cluster-master-1] Node 'opensearch-cluster-master-1' initialized
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:20,839][INFO ][o.o.c.r.a.AllocationService] [opensearch-cluster-master-2] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.opendistro_security][0]]]).
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:31,993][INFO ][o.o.i.i.ManagedIndexCoordinator] [opensearch-cluster-master-2] Performing move cluster state metadata.
opensearch-cluster-master-2 opensearch [2021-09-22T03:09:45,001][WARN ][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-2] health check of [/usr/share/opensearch/data/nodes/0] took [5204ms] which is above the warn threshold of [5s]
opensearch-cluster-master-0 opensearch [2021-09-22T03:10:23,012][ERROR][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-0] health check of [/usr/share/opensearch/data/nodes/0] failed
opensearch-cluster-master-0 opensearch java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/.opensearch_temp_file: Read-only file system
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.monitorFSHealth(FsHealthService.java:185) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.run(FsHealthService.java:160) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.threadpool.Scheduler$ReschedulingRunnable.doRun(Scheduler.java:226) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:23,741][INFO ][o.o.c.r.a.AllocationService] [opensearch-cluster-master-2] updating number_of_replicas to [1] for indices [.opendistro_security]
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:23,746][INFO ][o.o.c.r.a.AllocationService] [opensearch-cluster-master-2] Cluster health status changed from [YELLOW] to [GREEN] (reason: [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} reason: health check failed]).
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:23,750][INFO ][o.o.c.s.MasterService    ] [opensearch-cluster-master-2] node-left[{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} reason: health check failed], term: 3, version: 21, delta: removed {{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}}
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:24,406][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-2] master node changed {previous [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}], current []}, term: 3, version: 20, reason: becoming candidate: Publication.onCompletion(false)
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:24,425][WARN ][o.o.c.s.MasterService    ] [opensearch-cluster-master-2] failing [node-left[{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr} reason: health check failed]]: failed to commit cluster state version [21]
opensearch-cluster-master-2 opensearch org.opensearch.cluster.coordination.FailedToCommitClusterStateException: publication failed
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1482) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.action.ActionRunnable.onFailure(ActionRunnable.java:101) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.OpenSearchExecutors$DirectExecutorService.execute(OpenSearchExecutors.java:239) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:119) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:81) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1405) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:138) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:186) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication.access$500(Publication.java:55) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:381) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator$5.onFailure(Coordinator.java:1165) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler$PublicationContext$1.onFailure(PublicationTransportHandler.java:347) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler$PublicationContext.lambda$sendClusterState$2(PublicationTransportHandler.java:432) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler$PublicationContext$3.handleException(PublicationTransportHandler.java:450) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleException(SecurityInterceptor.java:294) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1217) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1326) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$DirectResponseChannel$2.run(TransportService.java:1305) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:697) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch Caused by: org.opensearch.common.util.concurrent.UncategorizedExecutionException: Failed execution
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.FutureUtils.rethrowExecutionException(FutureUtils.java:104) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.FutureUtils.get(FutureUtils.java:96) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:124) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 20 more
opensearch-cluster-master-2 opensearch Caused by: java.util.concurrent.ExecutionException: java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/_state/_k.fdm: Read-only file system
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.BaseFuture$Sync.getValue(BaseFuture.java:286) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:259) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:78) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.FutureUtils.get(FutureUtils.java:89) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:124) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 20 more
opensearch-cluster-master-2 opensearch Caused by: java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/_state/_k.fdm: Read-only file system
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:118) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.fieldsWriter(Lucene87StoredFieldsFormat.java:141) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.StoredFieldsConsumer.initStoredFieldsWriter(StoredFieldsConsumer.java:48) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.StoredFieldsConsumer.startDocument(StoredFieldsConsumer.java:55) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:449) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:485) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.PersistedClusterStateService$MetadataIndexWriter.updateIndexMetadataDocument(PersistedClusterStateService.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.PersistedClusterStateService$Writer.updateMetadata(PersistedClusterStateService.java:681) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.PersistedClusterStateService$Writer.writeIncrementalStateAndCommit(PersistedClusterStateService.java:615) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.GatewayMetaState$LucenePersistedState.setLastAcceptedState(GatewayMetaState.java:556) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.CoordinationState.handlePublishRequest(CoordinationState.java:370) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handlePublishRequest(Coordinator.java:350) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler.acceptState(PublicationTransportHandler.java:240) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler.handleIncomingPublishRequest(PublicationTransportHandler.java:223) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler.lambda$new$0(PublicationTransportHandler.java:115) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 3 more
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:24,443][ERROR][o.o.c.c.Coordinator      ] [opensearch-cluster-master-2] unexpected failure during [node-left]
opensearch-cluster-master-2 opensearch org.opensearch.cluster.coordination.FailedToCommitClusterStateException: publication failed
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1482) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.action.ActionRunnable.onFailure(ActionRunnable.java:101) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.OpenSearchExecutors$DirectExecutorService.execute(OpenSearchExecutors.java:239) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:119) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:81) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1405) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:138) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:186) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication.access$500(Publication.java:55) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:381) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator$5.onFailure(Coordinator.java:1165) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler$PublicationContext$1.onFailure(PublicationTransportHandler.java:347) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler$PublicationContext.lambda$sendClusterState$2(PublicationTransportHandler.java:432) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler$PublicationContext$3.handleException(PublicationTransportHandler.java:450) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleException(SecurityInterceptor.java:294) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1217) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1326) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$DirectResponseChannel$2.run(TransportService.java:1305) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:697) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch Caused by: org.opensearch.common.util.concurrent.UncategorizedExecutionException: Failed execution
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.FutureUtils.rethrowExecutionException(FutureUtils.java:104) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.FutureUtils.get(FutureUtils.java:96) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:124) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 20 more
opensearch-cluster-master-2 opensearch Caused by: java.util.concurrent.ExecutionException: java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/_state/_k.fdm: Read-only file system
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.BaseFuture$Sync.getValue(BaseFuture.java:286) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:259) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:78) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.FutureUtils.get(FutureUtils.java:89) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:124) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 20 more
opensearch-cluster-master-2 opensearch Caused by: java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/_state/_k.fdm: Read-only file system
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:410) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:406) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:254) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:118) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:130) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.fieldsWriter(Lucene87StoredFieldsFormat.java:141) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.StoredFieldsConsumer.initStoredFieldsWriter(StoredFieldsConsumer.java:48) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.StoredFieldsConsumer.startDocument(StoredFieldsConsumer.java:55) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:449) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:485) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1757) ~[lucene-core-8.8.2.jar:8.8.2 a92a05e195b775b30ca410bc0a26e8e79e7b3bfb - mdrob - 2021-04-06 16:33:27]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.PersistedClusterStateService$MetadataIndexWriter.updateIndexMetadataDocument(PersistedClusterStateService.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.PersistedClusterStateService$Writer.updateMetadata(PersistedClusterStateService.java:681) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.PersistedClusterStateService$Writer.writeIncrementalStateAndCommit(PersistedClusterStateService.java:615) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.gateway.GatewayMetaState$LucenePersistedState.setLastAcceptedState(GatewayMetaState.java:556) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.CoordinationState.handlePublishRequest(CoordinationState.java:370) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handlePublishRequest(Coordinator.java:350) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler.acceptState(PublicationTransportHandler.java:240) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler.handleIncomingPublishRequest(PublicationTransportHandler.java:223) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.cluster.coordination.PublicationTransportHandler.lambda$new$0(PublicationTransportHandler.java:115) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 3 more
opensearch-cluster-master-0 opensearch [2021-09-22T03:10:26,633][INFO ][o.o.c.c.Coordinator      ] [opensearch-cluster-master-0] master node [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] failed, restarting discovery
opensearch-cluster-master-0 opensearch org.opensearch.OpenSearchException: node [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] failed [3] consecutive checks
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.LeaderChecker$CheckScheduler$1.handleException(LeaderChecker.java:306) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleException(SecurityInterceptor.java:294) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1217) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.lambda$handleException$3(InboundHandler.java:290) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.OpenSearchExecutors$DirectExecutorService.execute(OpenSearchExecutors.java:239) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.handleException(InboundHandler.java:288) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.handlerResponseError(InboundHandler.java:280) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:144) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:102) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:713) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:155) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:130) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:95) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:87) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:271) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1533) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1282) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1329) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:508) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:447) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch Caused by: org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-2][10.42.8.149:9300][internal:coordination/fault_detection/leader_check]
opensearch-cluster-master-0 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: rejecting leader check from [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] sent to a node that is no longer the master
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.LeaderChecker.handleLeaderCheck(LeaderChecker.java:201) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.cluster.coordination.LeaderChecker.lambda$new$0(LeaderChecker.java:126) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:314) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:220) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:120) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:102) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:713) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:155) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:130) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:95) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:87) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:271) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1533) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1282) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1329) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:508) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:447) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]
opensearch-cluster-master-0 opensearch 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) ~[?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:10:26,669][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-0] master node changed {previous [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}], current []}, term: 3, version: 20, reason: becoming candidate: onLeaderFailure
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:27,096][INFO ][o.o.c.c.Coordinator      ] [opensearch-cluster-master-1] master node [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] failed, restarting discovery
opensearch-cluster-master-1 opensearch org.opensearch.OpenSearchException: node [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] failed [3] consecutive checks
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.LeaderChecker$CheckScheduler$1.handleException(LeaderChecker.java:306) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityInterceptor$RestoringTransportResponseHandler.handleException(SecurityInterceptor.java:294) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1217) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.lambda$handleException$3(InboundHandler.java:290) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.OpenSearchExecutors$DirectExecutorService.execute(OpenSearchExecutors.java:239) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.handleException(InboundHandler.java:288) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.handlerResponseError(InboundHandler.java:280) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:144) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:102) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:713) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:155) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:130) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:95) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:87) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:271) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1533) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1282) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1329) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:508) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:447) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-2][10.42.8.149:9300][internal:coordination/fault_detection/leader_check]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: rejecting leader check from [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] sent to a node that is no longer the master
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.LeaderChecker.handleLeaderCheck(LeaderChecker.java:201) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.LeaderChecker.lambda$new$0(LeaderChecker.java:126) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:314) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:153) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.handleRequest(InboundHandler.java:220) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.messageReceived(InboundHandler.java:120) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundHandler.inboundMessage(InboundHandler.java:102) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TcpTransport.inboundMessage(TcpTransport.java:713) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundPipeline.forwardFragments(InboundPipeline.java:155) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundPipeline.doHandleBytes(InboundPipeline.java:130) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.InboundPipeline.handleBytes(InboundPipeline.java:95) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:87) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:271) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1533) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1282) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1329) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:508) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:447) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]
opensearch-cluster-master-1 opensearch 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) ~[?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:27,204][INFO ][o.o.c.s.ClusterApplierService] [opensearch-cluster-master-1] master node changed {previous [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}], current []}, term: 3, version: 20, reason: becoming candidate: onLeaderFailure
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:28,154][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=3, optionalJoin=Optional[Join{term=4, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:28,750][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=4, optionalJoin=Optional[Join{term=5, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:29,549][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=5, optionalJoin=Optional[Join{term=6, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:30,369][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=6, optionalJoin=Optional[Join{term=7, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:31,202][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=7, optionalJoin=Optional[Join{term=8, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:32,380][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=8, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:32,999][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=9, optionalJoin=Optional[Join{term=10, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:33,520][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=10, optionalJoin=Optional[Join{term=11, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:34,408][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:34,908][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=11, optionalJoin=Optional[Join{term=12, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:35,901][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=12, optionalJoin=Optional[Join{term=13, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:10:36,664][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:36,911][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=13, optionalJoin=Optional[Join{term=14, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:37,163][WARN ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] last failed join attempt was 238ms ago, failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=13, optionalJoin=Optional[Join{term=14, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:37,186][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 15, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:42,128][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=14, optionalJoin=Optional[Join{term=15, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:42,344][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=15, optionalJoin=Optional[Join{term=16, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:42,503][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=16, optionalJoin=Optional[Join{term=17, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:43,934][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=17, optionalJoin=Optional[Join{term=18, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:44,433][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:45,087][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=18, optionalJoin=Optional[Join{term=19, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:46,489][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=19, optionalJoin=Optional[Join{term=20, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:10:46,667][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:47,189][WARN ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] last failed join attempt was 696ms ago, failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=19, optionalJoin=Optional[Join{term=20, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:47,205][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 21, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:48,895][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=20, optionalJoin=Optional[Join{term=21, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:51,707][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=21, optionalJoin=Optional[Join{term=22, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:53,500][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=22, optionalJoin=Optional[Join{term=23, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:10:54,444][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:54,642][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=23, optionalJoin=Optional[Join{term=24, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:56,241][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=24, optionalJoin=Optional[Join{term=25, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:10:56,670][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:56,699][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=25, optionalJoin=Optional[Join{term=26, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:57,208][WARN ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] last failed join attempt was 504ms ago, failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=25, optionalJoin=Optional[Join{term=26, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:10:57,218][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 27, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:04,450][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-0 opensearch [2021-09-22T03:11:06,672][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:14,455][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-0 opensearch [2021-09-22T03:11:16,675][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:22,863][WARN ][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-1] health check of [/usr/share/opensearch/data/nodes/0] took [17011ms] which is above the warn threshold of [5s]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:22,919][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=26, optionalJoin=Optional[Join{term=27, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:22,929][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 28, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:23,006][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=27, optionalJoin=Optional[Join{term=28, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:24,462][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-0 opensearch [2021-09-22T03:11:26,683][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:25,554][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=28, optionalJoin=Optional[Join{term=29, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:27,919][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=29, optionalJoin=Optional[Join{term=30, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:31,425][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=30, optionalJoin=Optional[Join{term=31, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:34,469][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:32,931][WARN ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] last failed join attempt was 1.5s ago, failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=30, optionalJoin=Optional[Join{term=31, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:32,936][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 32, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:35,824][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=31, optionalJoin=Optional[Join{term=32, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:36,384][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=32, optionalJoin=Optional[Join{term=33, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:11:36,685][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:42,938][WARN ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] last failed join attempt was 6.5s ago, failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=32, optionalJoin=Optional[Join{term=33, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:44,472][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.12.53:9300] from hosts providers and [{opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}] from last-known cluster state; node term 3, last-accepted version 20 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:45,004][ERROR][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-2] health check of [/usr/share/opensearch/data/nodes/0] failed
opensearch-cluster-master-2 opensearch java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/.opensearch_temp_file: Read-only file system
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.monitorFSHealth(FsHealthService.java:185) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.run(FsHealthService.java:160) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.threadpool.Scheduler$ReschedulingRunnable.doRun(Scheduler.java:226) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:11:46,688][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:47,579][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:47,581][INFO ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=33, optionalJoin=Optional[Join{term=34, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) [opensearch-index-management-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) [opensearch-security-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) [opensearch-performance-analyzer-1.0.1.0.jar:1.0.1.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:11:54,476][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:11:56,692][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:57,582][WARN ][o.o.c.c.JoinHelper       ] [opensearch-cluster-master-1] last failed join attempt was 9.9s ago, failed to join {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr} with JoinRequest{sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, minimumTerm=33, optionalJoin=Optional[Join{term=34, lastAcceptedTerm=3, lastAcceptedVersion=21, sourceNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, targetNode={opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}}]}
opensearch-cluster-master-1 opensearch org.opensearch.transport.RemoteTransportException: [opensearch-cluster-master-1][10.42.12.53:9300][internal:cluster/coordination/join]
opensearch-cluster-master-1 opensearch Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:478) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:546) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.lambda$handleJoinRequest$7(Coordinator.java:509) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.action.ActionListener$1.onResponse(ActionListener.java:76) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:393) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService.connectToNode(TransportService.java:380) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:496) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:154) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.indexmanagement.rollup.interceptor.RollupInterceptor$interceptHandler$1.messageReceived(RollupInterceptor.kt:138) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceivedDecorate(SecuritySSLRequestHandler.java:193) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.transport.SecurityRequestHandler.messageReceivedDecorate(SecurityRequestHandler.java:173) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.ssl.transport.SecuritySSLRequestHandler.messageReceived(SecuritySSLRequestHandler.java:97) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.security.OpenSearchSecurityPlugin$7$1.messageReceived(OpenSearchSecurityPlugin.java:647) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportRequestHandler.messageReceived(PerformanceAnalyzerTransportRequestHandler.java:64) ~[?:?]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:85) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.transport.TransportService$8.doRun(TransportService.java:845) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-1 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-1 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-1 opensearch [2021-09-22T03:11:57,599][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:12:04,482][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:06,697][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:12:07,602][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:12:14,485][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:16,699][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:12:17,605][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:23,019][ERROR][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-0] health check of [/usr/share/opensearch/data/nodes/0] failed
opensearch-cluster-master-0 opensearch java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/.opensearch_temp_file: Read-only file system
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.monitorFSHealth(FsHealthService.java:185) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.run(FsHealthService.java:160) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.threadpool.Scheduler$ReschedulingRunnable.doRun(Scheduler.java:226) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:12:24,489][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:26,701][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:12:27,610][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:12:34,492][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:36,707][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:12:37,613][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:12:44,495][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:46,716][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:12:47,616][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:12:54,500][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:12:56,722][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:12:57,619][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:01,603][INFO ][o.o.j.s.JobSweeper       ] [opensearch-cluster-master-2] Running full sweep
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:04,503][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:06,725][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:06,753][INFO ][o.o.j.s.JobSweeper       ] [opensearch-cluster-master-0] Running full sweep
opensearch-cluster-master-1 opensearch [2021-09-22T03:13:07,623][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:14,506][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:16,726][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:13:17,627][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:24,508][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:26,735][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:13:27,633][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:34,511][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:36,737][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:13:37,637][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:44,520][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:45,011][ERROR][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-2] health check of [/usr/share/opensearch/data/nodes/0] failed
opensearch-cluster-master-2 opensearch java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/.opensearch_temp_file: Read-only file system
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.monitorFSHealth(FsHealthService.java:185) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.run(FsHealthService.java:160) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.threadpool.Scheduler$ReschedulingRunnable.doRun(Scheduler.java:226) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:45,142][WARN ][o.o.i.f.SyncedFlushService] [opensearch-cluster-master-2] failed to flush shard [.opendistro_security][0], node[61sMC4gJQCiPVS0y-UMGJw], [R], s[STARTED], a[id=bPlD8bDaQPOrfpFDGLL47g] on inactive
opensearch-cluster-master-2 opensearch org.opensearch.index.engine.FlushFailedEngineException: Flush failed
opensearch-cluster-master-2 opensearch 	at org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1882) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.index.shard.IndexShard.flush(IndexShard.java:1116) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.indices.flush.SyncedFlushService$2.doRun(SyncedFlushService.java:162) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch Caused by: java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/indices/oqAdL1F-TbeZVzCwElOr9Q/0/translog/translog-7926379507485154385.ckp: Read-only file system
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.newByteChannel(Files.java:375) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.createFile(Files.java:652) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.TempFileHelper.create(TempFileHelper.java:137) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:160) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.createTempFile(Files.java:872) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.index.translog.Translog.copyCheckpointTo(Translog.java:287) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.index.translog.Translog.rollGeneration(Translog.java:1656) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1866) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	... 7 more
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:46,740][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:13:47,640][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:13:54,524][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:13:56,742][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:13:57,644][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:14:04,527][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:06,745][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:07,647][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:07,716][INFO ][o.o.j.s.JobSweeper       ] [opensearch-cluster-master-1] Running full sweep
opensearch-cluster-master-2 opensearch [2021-09-22T03:14:14,535][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:16,749][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:17,650][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:18,303][WARN ][o.o.i.f.SyncedFlushService] [opensearch-cluster-master-0] failed to flush shard [.opendistro_security][0], node[PRO48_2LToSUQX8dSnZZnA], [P], s[STARTED], a[id=KmJxNSJmSL25v0sMxvhFCw] on inactive
opensearch-cluster-master-0 opensearch org.opensearch.index.engine.FlushFailedEngineException: Flush failed
opensearch-cluster-master-0 opensearch 	at org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1882) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.index.shard.IndexShard.flush(IndexShard.java:1116) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.indices.flush.SyncedFlushService$2.doRun(SyncedFlushService.java:162) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch Caused by: java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/indices/oqAdL1F-TbeZVzCwElOr9Q/0/translog/translog-13879303083707416115.ckp: Read-only file system
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.Files.newByteChannel(Files.java:375) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.Files.createFile(Files.java:652) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.TempFileHelper.create(TempFileHelper.java:137) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:160) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.Files.createTempFile(Files.java:872) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.index.translog.Translog.copyCheckpointTo(Translog.java:287) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.index.translog.Translog.rollGeneration(Translog.java:1656) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.index.engine.InternalEngine.flush(InternalEngine.java:1866) ~[opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	... 7 more
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:23,031][ERROR][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-0] health check of [/usr/share/opensearch/data/nodes/0] failed
opensearch-cluster-master-0 opensearch java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/.opensearch_temp_file: Read-only file system
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-0 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-0 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-0 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.monitorFSHealth(FsHealthService.java:185) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.run(FsHealthService.java:160) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.threadpool.Scheduler$ReschedulingRunnable.doRun(Scheduler.java:226) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-0 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-0 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-2 opensearch [2021-09-22T03:14:24,538][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:26,752][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:27,654][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:14:34,545][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:36,759][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:37,657][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:14:44,549][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:46,761][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:47,663][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:14:54,551][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:14:56,763][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:14:57,671][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:15:04,554][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:15:06,765][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:15:07,676][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:15:14,562][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:15:16,779][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:15:17,679][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:15:24,567][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:15:26,782][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:15:27,682][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.8.149:9300, 10.42.0.146:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-1 opensearch [2021-09-22T03:15:32,620][WARN ][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-1] health check of [/usr/share/opensearch/data/nodes/0] took [6217ms] which is above the warn threshold of [5s]
opensearch-cluster-master-2 opensearch [2021-09-22T03:15:34,573][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-0 opensearch [2021-09-22T03:15:36,788][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:15:37,686][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3
opensearch-cluster-master-2 opensearch [2021-09-22T03:15:44,577][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-2] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-2 opensearch [2021-09-22T03:15:45,016][ERROR][o.o.m.f.FsHealthService  ] [opensearch-cluster-master-2] health check of [/usr/share/opensearch/data/nodes/0] failed
opensearch-cluster-master-2 opensearch java.nio.file.FileSystemException: /usr/share/opensearch/data/nodes/0/.opensearch_temp_file: Read-only file system
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:100) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
opensearch-cluster-master-2 opensearch 	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:478) ~[?:?]
opensearch-cluster-master-2 opensearch 	at java.nio.file.Files.newOutputStream(Files.java:224) ~[?:?]
opensearch-cluster-master-2 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.monitorFSHealth(FsHealthService.java:185) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.monitor.fs.FsHealthService$FsHealthMonitor.run(FsHealthService.java:160) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.threadpool.Scheduler$ReschedulingRunnable.doRun(Scheduler.java:226) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:756) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:50) [opensearch-1.0.0.jar:1.0.0]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
opensearch-cluster-master-2 opensearch 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
opensearch-cluster-master-2 opensearch 	at java.lang.Thread.run(Thread.java:832) [?:?]
opensearch-cluster-master-0 opensearch [2021-09-22T03:15:46,790][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-0] this node is unhealthy: health check failed on [/usr/share/opensearch/data/nodes/0]
opensearch-cluster-master-1 opensearch [2021-09-22T03:15:47,688][WARN ][o.o.c.c.ClusterFormationFailureHelper] [opensearch-cluster-master-1] master not discovered or elected yet, an election requires at least 2 nodes with ids from [PRO48_2LToSUQX8dSnZZnA, pTorOyp-S6uhV4GkS1dklA, 61sMC4gJQCiPVS0y-UMGJw], have discovered [{opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}, {opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-0}{PRO48_2LToSUQX8dSnZZnA}{X-lsJf0PTO-CTbnMBaIAQA}{10.42.0.146}{10.42.0.146:9300}{dimr}] which is a quorum; discovery will continue using [10.42.0.146:9300, 10.42.8.149:9300] from hosts providers and [{opensearch-cluster-master-2}{61sMC4gJQCiPVS0y-UMGJw}{9yIAvnTdSeOjRVLZxabEPw}{10.42.8.149}{10.42.8.149:9300}{dimr}, {opensearch-cluster-master-1}{pTorOyp-S6uhV4GkS1dklA}{8EFd5IZNRlK-hdAVnsKC_A}{10.42.12.53}{10.42.12.53:9300}{dimr}] from last-known cluster state; node term 35, last-accepted version 21 in term 3

values.yaml

---
clusterName: "opensearch-cluster"
nodeGroup: "master"

# The service that non master groups will try to connect to when joining the cluster
# This should be set to clusterName + "-" + nodeGroup for your master group
masterService: "opensearch-cluster-master"

# OpenSearch roles that will be applied to this nodeGroup
# These will be set as environment variables. E.g. node.master=true
roles:
  master: "true"
  ingest: "true"
  data: "true"
  remote_cluster_client: "true"

replicas: 3
minimumMasterNodes: 1

majorVersion: ""

# Allows you to add any config files in {{ .Values.opensearchHome }}/config
opensearchHome: /usr/share/opensearch
# such as opensearch.yml and log4j2.properties
config:
  opensearch.yml:
    cluster.name: opensearch-cluster

    # Bind to all interfaces because we don't know what IP address Docker will assign to us.
    network.host: 0.0.0.0

    # # minimum_master_nodes need to be explicitly set when bound on a public IP
    # # set to 1 to allow single node clusters
    discovery.zen.minimum_master_nodes: 1

    # Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
    # discovery.type: single-node

    ######## Start OpenSearch Security Demo Configuration ########
    # WARNING: revise all the lines below before you go into production
    plugins:
      security:
        ssl:
          transport:
            pemcert_filepath: esnode.pem
            pemkey_filepath: esnode-key.pem
            pemtrustedcas_filepath: root-ca.pem
            enforce_hostname_verification: false
          http:
            enabled: false
            pemcert_filepath: esnode.pem
            pemkey_filepath: esnode-key.pem
            pemtrustedcas_filepath: root-ca.pem
        allow_unsafe_democertificates: true
        allow_default_init_securityindex: true
        authcz:
          admin_dn:
            - CN=kirk,OU=client,O=client,L=test, C=de
        audit.type: internal_opensearch
        enable_snapshot_restore_privilege: true
        check_snapshot_restore_write_privileges: true
        restapi:
          roles_enabled: ["all_access", "security_rest_api_access"]
        system_indices:
          enabled: true
          indices:
            [
              ".opendistro-alerting-config",
              ".opendistro-alerting-alert*",
              ".opendistro-anomaly-results*",
              ".opendistro-anomaly-detector*",
              ".opendistro-anomaly-checkpoints",
              ".opendistro-anomaly-detection-state",
              ".opendistro-reports-*",
              ".opendistro-notifications-*",
              ".opendistro-notebooks",
              ".opendistro-asynchronous-search-response*",
            ]
    ######## End OpenSearch Security Demo Configuration ########
  # log4j2.properties:

# Extra environment variables to append to this nodeGroup
# This will be appended to the current 'env:' key. You can use any of the kubernetes env
# syntax here
extraEnvs: []
#  - name: MY_ENVIRONMENT_VAR
#    value: the_value_goes_here

# Allows you to load environment variables from kubernextes secret or config map
envFrom: []
# - secretRef:
#     name: env-secret
# - configMapRef:
#     name: config-map

# A list of secrets and their paths to mount inside the pod
# This is useful for mounting certificates for security and for mounting
# the X-Pack license
secretMounts: []

hostAliases: []
#- ip: "127.0.0.1"
#  hostnames:
#  - "foo.local"
#  - "bar.local"

image: "alvarogg777/opensearch"
# override image tag, which is .Chart.AppVersion by default
imageTag: "1.0.1"
imagePullPolicy: "IfNotPresent"

podAnnotations: {}
  # iam.amazonaws.com/role: es-cluster

# additionals labels
labels: {}

opensearchJavaOpts: "-Xmx512M -Xms512M"

resources:
  requests:
    cpu: "1000m"
    memory: "100Mi"

initResources: {}
  # limits:
  #   cpu: "25m"
  #   # memory: "128Mi"
  # requests:
  #   cpu: "25m"
  #   memory: "128Mi"

sidecarResources: {}
  # limits:
  #   cpu: "25m"
  #   # memory: "128Mi"
  # requests:
  #   cpu: "25m"
  #   memory: "128Mi"

networkHost: "0.0.0.0"

rbac:
  create: false
  serviceAccountAnnotations: {}
  serviceAccountName: ""

podSecurityPolicy:
  create: false
  name: ""
  spec:
    privileged: true
    fsGroup:
      rule: RunAsAny
    runAsUser:
      rule: RunAsAny
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
      - secret
      - configMap
      - persistentVolumeClaim
      - emptyDir

persistence:
  enabled: true
  labels:
    # Add default labels for the volumeClaimTemplate of the StatefulSet
    enabled: false
  ## OpenSearch Persistent Volume Storage Class
  ## If defined, storageClassName: <storageClass>
  ## If set to "-", storageClassName: "", which disables dynamic provisioning
  ## If undefined (the default) or set to null, no storageClassName spec is
  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
  ##   GKE, AWS & OpenStack)
  ##
  storageClass: "longhorn"
  accessModes:
    - ReadWriteOnce
  size: 8Gi
  annotations: {}

extraVolumes: []
  # - name: extras
  #   emptyDir: {}

extraVolumeMounts: []
  # - name: extras
  #   mountPath: /usr/share/extras
  #   readOnly: true

extraContainers: []
  # - name: do-something
  #   image: busybox
  #   command: ['do', 'something']

extraInitContainers: []
  # - name: do-somethings
  #   image: busybox
  #   command: ['do', 'something']

# This is the PriorityClass settings as defined in
# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
priorityClassName: ""

# By default this will make sure two pods don't end up on the same node
# Changing this to a region would allow you to spread pods across regions
antiAffinityTopologyKey: "kubernetes.io/hostname"

# Hard means that by default pods will only be scheduled if there are enough nodes for them
# and that they will never end up on the same node. Setting this to soft will do this "best effort"
antiAffinity: "soft"

# This is the node affinity settings as defined in
# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature
nodeAffinity: {}

# The default is to deploy all pods serially. By setting this to parallel all pods are started at
# the same time when bootstrapping the cluster
podManagementPolicy: "Parallel"

# The environment variables injected by service links are not used, but can lead to slow OpenSearch boot times when
# there are many services in the current namespace.
# If you experience slow pod startups you probably want to set this to `false`.
enableServiceLinks: true

protocol: http
httpPort: 9200
transportPort: 9300

service:
  labels: {}
  labelsHeadless: {}
  type: ClusterIP
  nodePort: ""
  annotations: {}
  traefik.ingress.kubernetes.io/service.serversscheme: https
  httpPortName: http
  transportPortName: transport
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  externalTrafficPolicy: ""

updateStrategy: RollingUpdate

# This is the max unavailable setting for the pod disruption budget
# The default value of 1 will make sure that kubernetes won't allow more than 1
# of your pods to be unavailable during maintenance
maxUnavailable: 1

podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000

securityContext:
  capabilities:
    drop:
      - ALL
  # readOnlyRootFilesystem: true
  runAsNonRoot: true
  runAsUser: 1000

securityConfig:
  enabled: true
  path: "/usr/share/opensearch/plugins/opensearch-security/securityconfig"
  actionGroupsSecret:
  configSecret:
  internalUsersSecret:
  rolesSecret:
  rolesMappingSecret:
  tenantsSecret:
  #The following option simplifies securityConfig by using a single secret and specifying the respective secrets in the corresponding files instead of creating different secrets for config,internal users, roles, roles mapping and tenants
  #Note that this is an alternative to the above secrets and shouldn't be used if the above secrets are used
  config:
    securityConfigSecret: mysecret
    data:
      audit.yml: |-
        _meta:
          type: "audit"
          config_version: 2

        config:
          # enable/disable audit logging
          enabled: true

          audit:
            # Enable/disable REST API auditing
            enable_rest: true

            # Categories to exclude from REST API auditing
            disabled_rest_categories:
              - AUTHENTICATED
              - GRANTED_PRIVILEGES

            # Enable/disable Transport API auditing
            enable_transport: true

            # Categories to exclude from Transport API auditing
            disabled_transport_categories:
              - AUTHENTICATED
              - GRANTED_PRIVILEGES

            # Users to be excluded from auditing. Wildcard patterns are supported. Eg:
            # ignore_users: ["test-user", "employee-*"]
            ignore_users:
              - kibanaserver

            # Requests to be excluded from auditing. Wildcard patterns are supported. Eg:
            # ignore_requests: ["indices:data/read/*", "SearchRequest"]
            ignore_requests: []

            # Log individual operations in a bulk request
            resolve_bulk_requests: false

            # Include the body of the request (if available) for both REST and the transport layer
            log_request_body: true

            # Logs all indices affected by a request. Resolves aliases and wildcards/date patterns
            resolve_indices: true

            # Exclude sensitive headers from being included in the logs. Eg: Authorization
            exclude_sensitive_headers: true

          compliance:
            # enable/disable compliance
            enabled: true

            # Log updates to internal security changes
            internal_config: true

            # Log external config files for the node
            external_config: false

            # Log only metadata of the document for read events
            read_metadata_only: true

            # Map of indexes and fields to monitor for read events. Wildcard patterns are supported for both index names and fields. Eg:
            # read_watched_fields: {
            #   "twitter": ["message"]
            #   "logs-*": ["id", "attr*"]
            # }
            read_watched_fields: {}

            # List of users to ignore for read events. Wildcard patterns are supported. Eg:
            # read_ignore_users: ["test-user", "employee-*"]
            read_ignore_users:
              - kibanaserver

            # Log only metadata of the document for write events
            write_metadata_only: true

            # Log only diffs for document updates
            write_log_diffs: false

            # List of indices to watch for write events. Wildcard patterns are supported
            # write_watched_indices: ["twitter", "logs-*"]
            write_watched_indices: []

            # List of users to ignore for write events. Wildcard patterns are supported. Eg:
            # write_ignore_users: ["test-user", "employee-*"]
            write_ignore_users:
              - kibanaserver

      opensearch.yml: |-
        cluster.name: opensearch-cluster

        # Bind to all interfaces because we don't know what IP address Docker will assign to us.
        network.host: 0.0.0.0

        # # minimum_master_nodes need to be explicitly set when bound on a public IP
        # # set to 1 to allow single node clusters
        discovery.zen.minimum_master_nodes: 1

        # Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
        # discovery.type: single-node

        ######## Start OpenSearch Security Demo Configuration ########
        # WARNING: revise all the lines below before you go into production
        plugins:
          security:
            ssl:
              transport:
                pemcert_filepath: esnode.pem
                pemkey_filepath: esnode-key.pem
                pemtrustedcas_filepath: root-ca.pem
                enforce_hostname_verification: false
              http:
                enabled: false
                pemcert_filepath: esnode.pem
                pemkey_filepath: esnode-key.pem
                pemtrustedcas_filepath: root-ca.pem
            allow_unsafe_democertificates: true
            allow_default_init_securityindex: true
            authcz:
              admin_dn:
                - CN=kirk,OU=client,O=client,L=test, C=de
            audit.type: internal_opensearch
            enable_snapshot_restore_privilege: true
            check_snapshot_restore_write_privileges: true
            restapi:
              roles_enabled: ["all_access", "security_rest_api_access"]
            system_indices:
              enabled: true
              indices:
                [
                  ".opendistro-alerting-config",
                  ".opendistro-alerting-alert*",
                  ".opendistro-anomaly-results*",
                  ".opendistro-anomaly-detector*",
                  ".opendistro-anomaly-checkpoints",
                  ".opendistro-anomaly-detection-state",
                  ".opendistro-reports-*",
                  ".opendistro-notifications-*",
                  ".opendistro-notebooks",
                  ".opendistro-asynchronous-search-response*",
                ]
      config.yml: |-
        ---

        # This is the main OpenSearch Security configuration file where authentication
        # and authorization is defined.
        #
        # You need to configure at least one authentication domain in the authc of this file.
        # An authentication domain is responsible for extracting the user credentials from
        # the request and for validating them against an authentication backend like Active Directory for example.
        #
        # If more than one authentication domain is configured the first one which succeeds wins.
        # If all authentication domains fail then the request is unauthenticated.
        # In this case an exception is thrown and/or the HTTP status is set to 401.
        #
        # After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect
        # the roles from a given backend for the authenticated user.
        #
        # Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both.
        #        http_enabled: true
        #        transport_enabled: true
        #
        # For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to
        # find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated.
        # If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous"
        # and one role named "anonymous_backendrole".
        # If you enable anonymous authentication all HTTP authenticators will not challenge.
        #
        #
        # Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert"
        # first and the challenging one last.
        # Because it's not possible to challenge a client with two different authentication methods (for example
        # Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation
        # by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request.
        #
        # Default value of the challenge flag is true.
        #
        #
        # HTTP
        #   basic (challenging)
        #   proxy (not challenging, needs xff)
        #   kerberos (challenging)
        #   clientcert (not challenging, needs https)
        #   jwt (not challenging)
        #   host (not challenging) #DEPRECATED, will be removed in a future version.
        #                          host based authentication is configurable in roles_mapping

        # Authc
        #   internal
        #   noop
        #   ldap

        # Authz
        #   ldap
        #   noop



        _meta:
          type: "config"
          config_version: 2

        config:
          dynamic:
            # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index
            # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default)
            # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently
            #filtered_alias_mode: warn
            #do_not_fail_on_forbidden: false
            #kibana:
            # Kibana multitenancy
            #multitenancy_enabled: true
            #server_username: kibanaserver
            #index: '.kibana'
            http:
              anonymous_auth_enabled: false
              xff:
                enabled: false
                internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern
                #internalProxies: '.*' # trust all internal proxies, regex pattern
                #remoteIpHeader:  'x-forwarded-for'
                ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help
                ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For
                ###### and here https://tools.ietf.org/html/rfc7239
                ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve
            authc:
              kerberos_auth_domain:
                http_enabled: false
                transport_enabled: false
                order: 6
                http_authenticator:
                  type: kerberos
                  challenge: true
                  config:
                    # If true a lot of kerberos/security related debugging output will be logged to standard out
                    krb_debug: false
                    # If true then the realm will be stripped from the user name
                    strip_realm_from_principal: true
                authentication_backend:
                  type: noop
              basic_internal_auth_domain:
                description: "Authenticate via HTTP Basic against internal users database"
                http_enabled: true
                transport_enabled: true
                order: 4
                http_authenticator:
                  type: basic
                  challenge: true
                authentication_backend:
                  type: intern
              proxy_auth_domain:
                description: "Authenticate via proxy"
                http_enabled: false
                transport_enabled: false
                order: 3
                http_authenticator:
                  type: proxy
                  challenge: false
                  config:
                    user_header: "x-proxy-user"
                    roles_header: "x-proxy-roles"
                authentication_backend:
                  type: noop
              jwt_auth_domain:
                description: "Authenticate via Json Web Token"
                http_enabled: false
                transport_enabled: false
                order: 0
                http_authenticator:
                  type: jwt
                  challenge: false
                  config:
                    signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key"
                    jwt_header: "Authorization"
                    jwt_url_parameter: null
                    roles_key: null
                    subject_key: null
                authentication_backend:
                  type: noop
              clientcert_auth_domain:
                description: "Authenticate via SSL client certificates"
                http_enabled: false
                transport_enabled: false
                order: 2
                http_authenticator:
                  type: clientcert
                  config:
                    username_attribute: cn #optional, if omitted DN becomes username
                  challenge: false
                authentication_backend:
                  type: noop
              ldap:
                description: "Authenticate via LDAP or Active Directory"
                http_enabled: false
                transport_enabled: false
                order: 5
                http_authenticator:
                  type: basic
                  challenge: false
                authentication_backend:
                  # LDAP authentication backend (authenticate users against a LDAP or Active Directory)
                  type: ldap
                  config:
                    # enable ldaps
                    enable_ssl: false
                    # enable start tls, enable_ssl should be false
                    enable_start_tls: false
                    # send client certificate
                    enable_ssl_client_auth: false
                    # verify ldap hostname
                    verify_hostnames: true
                    hosts:
                    - localhost:8389
                    bind_dn: null
                    password: null
                    userbase: 'ou=people,dc=example,dc=com'
                    # Filter to search for users (currently in the whole subtree beneath userbase)
                    # {0} is substituted with the username
                    usersearch: '(sAMAccountName={0})'
                    # Use this attribute from the user as username (if not set then DN is used)
                    username_attribute: null
            authz:
              roles_from_myldap:
                description: "Authorize via LDAP or Active Directory"
                http_enabled: false
                transport_enabled: false
                authorization_backend:
                  # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too)
                  type: ldap
                  config:
                    # enable ldaps
                    enable_ssl: false
                    # enable start tls, enable_ssl should be false
                    enable_start_tls: false
                    # send client certificate
                    enable_ssl_client_auth: false
                    # verify ldap hostname
                    verify_hostnames: true
                    hosts:
                    - localhost:8389
                    bind_dn: null
                    password: null
                    rolebase: 'ou=groups,dc=example,dc=com'
                    # Filter to search for roles (currently in the whole subtree beneath rolebase)
                    # {0} is substituted with the DN of the user
                    # {1} is substituted with the username
                    # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute
                    rolesearch: '(member={0})'
                    # Specify the name of the attribute which value should be substituted with {2} above
                    userroleattribute: null
                    # Roles as an attribute of the user entry
                    userrolename: disabled
                    #userrolename: memberOf
                    # The attribute in a role entry containing the name of that role, Default is "name".
                    # Can also be "dn" to use the full DN as rolename.
                    rolename: cn
                    # Resolve nested roles transitive (roles which are members of other roles and so on ...)
                    resolve_nested_roles: true
                    userbase: 'ou=people,dc=example,dc=com'
                    # Filter to search for users (currently in the whole subtree beneath userbase)
                    # {0} is substituted with the username
                    usersearch: '(uid={0})'
                    # Skip users matching a user name, a wildcard or a regex pattern
                    #skip_users:
                    #  - 'cn=Michael Jackson,ou*people,o=TEST'
                    #  - '/\S*/'
              roles_from_another_ldap:
                description: "Authorize via another Active Directory"
                http_enabled: false
                transport_enabled: false
                authorization_backend:
                  type: ldap
                  #config goes here ...
          #    auth_failure_listeners:
          #      ip_rate_limiting:
          #        type: ip
          #        allowed_tries: 10
          #        time_window_seconds: 3600
          #        block_expiry_seconds: 600
          #        max_blocked_clients: 100000
          #        max_tracked_clients: 100000
          #      internal_authentication_backend_limiting:
          #        type: username
          #        authentication_backend: intern
          #        allowed_tries: 10
          #        time_window_seconds: 3600
          #        block_expiry_seconds: 600
          #        max_blocked_clients: 100000
          #        max_tracked_clients: 100000

      internal_users.yml: |-
        # This is the internal user database
        # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh

        _meta:
          type: "internalusers"
          config_version: 2

        # Define your internal users here
        boro:
          hash: ommited
          reserved: false
          hidden: false
          # opensearch_security_roles:
          # - "admin"
          backend_roles:
          - "admin"
          static: false

        ## Demo users

        admin:
          hash: ommited
          reserved: true
          backend_roles:
          - "admin"
          description: "Demo admin user"

        kibanaserver:
          hash: ommitted
          reserved: true
          description: "Demo user for the OpenSearch Dashboards server"

        kibanaro:
          hash: ommitted
          reserved: false
          backend_roles:
          - "kibanauser"
          - "readall"
          attributes:
            attribute1: "value1"
            attribute2: "value2"
            attribute3: "value3"
          description: "Demo read-only user for OpenSearch dashboards"

        logstash:
          hash: ommitted
          reserved: false
          backend_roles:
          - "logstash"
          description: "Demo logstash user"

        readall:
          hash: ommitted
          reserved: false
          backend_roles:
          - "readall"
          description: "Demo readall user"

        snapshotrestore:
          hash: ommitted
          reserved: false
          backend_roles:
          - "snapshotrestore"
          description: "Demo snapshotrestore user"

      nodes_dn.yml: |-
        _meta:
          type: "nodesdn"
          config_version: 2

        # Define nodesdn mapping name and corresponding values
        # cluster1:
        #   nodes_dn:
        #       - CN=*.example.com

      roles.yml: |-
        _meta:
          type: "roles"
          config_version: 2

        # Restrict users so they can only view visualization and dashboard on OpenSearchDashboards
        kibana_read_only:
          reserved: true

        # The security REST API access role is used to assign specific users access to change the security settings through the REST API.
        security_rest_api_access:
          reserved: true

        # Allows users to view monitors, destinations and alerts
        alerting_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/get'
            - 'cluster:admin/opendistro/alerting/destination/get'
            - 'cluster:admin/opendistro/alerting/monitor/get'
            - 'cluster:admin/opendistro/alerting/monitor/search'

        # Allows users to view and acknowledge alerts
        alerting_ack_alerts:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/alerting/alerts/*'

        # Allows users to use all alerting functionality
        alerting_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/alerting/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'

        # Allow users to read Anomaly Detection detectors and results
        anomaly_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/ad/detector/info'
            - 'cluster:admin/opendistro/ad/detector/search'
            - 'cluster:admin/opendistro/ad/detectors/get'
            - 'cluster:admin/opendistro/ad/result/search'
            - 'cluster:admin/opendistro/ad/tasks/search'

        # Allows users to use all Anomaly Detection functionality
        anomaly_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster_monitor'
            - 'cluster:admin/opendistro/ad/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices_monitor'
                - 'indices:admin/aliases/get'
                - 'indices:admin/mappings/get'

        # Allows users to read Notebooks
        notebooks_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/list'
            - 'cluster:admin/opendistro/notebooks/get'

        # Allows users to all Notebooks functionality
        notebooks_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/notebooks/create'
            - 'cluster:admin/opendistro/notebooks/update'
            - 'cluster:admin/opendistro/notebooks/delete'
            - 'cluster:admin/opendistro/notebooks/get'
            - 'cluster:admin/opendistro/notebooks/list'

        # Allows users to read and download Reports
        reports_instances_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'

        # Allows users to read and download Reports and Report-definitions
        reports_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'

        # Allows users to all Reports functionality
        reports_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/reports/definition/create'
            - 'cluster:admin/opendistro/reports/definition/update'
            - 'cluster:admin/opendistro/reports/definition/on_demand'
            - 'cluster:admin/opendistro/reports/definition/delete'
            - 'cluster:admin/opendistro/reports/definition/get'
            - 'cluster:admin/opendistro/reports/definition/list'
            - 'cluster:admin/opendistro/reports/instance/list'
            - 'cluster:admin/opendistro/reports/instance/get'
            - 'cluster:admin/opendistro/reports/menu/download'

        # Allows users to use all asynchronous-search functionality
        asynchronous_search_full_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/*'
          index_permissions:
            - index_patterns:
                - '*'
              allowed_actions:
                - 'indices:data/read/search*'

        # Allows users to read stored asynchronous-search results
        asynchronous_search_read_access:
          reserved: true
          cluster_permissions:
            - 'cluster:admin/opendistro/asynchronous_search/get'

      roles_mapping.yml: |-
        ---
        # In this file users, backendroles and hosts can be mapped to Security roles.
        # Permissions for OpenSearch roles are configured in roles.yml

        _meta:
          type: "rolesmapping"
          config_version: 2

        # Define your roles mapping here

        ## Demo roles mapping

        all_access:
          reserved: false
          backend_roles:
          - "admin"
          description: "Maps admin to all_access"

        own_index:
          reserved: false
          users:
          - "*"
          description: "Allow full access to an index named like the username"

        logstash:
          reserved: false
          backend_roles:
          - "logstash"

        kibana_user:
          reserved: false
          backend_roles:
          - "kibanauser"
          description: "Maps kibanauser to kibana_user"

        readall:
          reserved: false
          backend_roles:
          - "readall"

        manage_snapshots:
          reserved: false
          backend_roles:
          - "snapshotrestore"

        kibana_server:
          reserved: true
          users:
          - "kibanaserver"


      tenants.yml: |-
        ---
        _meta:
          type: "tenants"
          config_version: 2

        # Define your tenants here

        ## Demo tenants
        admin_tenant:
          reserved: false
          description: "Demo tenant for admin user"

      action_groups.yml: |-
          _meta:
            type: "actiongroups"
            config_version: 2

      whitelist.yml: |-
        ---
        _meta:
          type: "whitelist"
          config_version: 2

        # Description:
        # enabled - feature flag.
        # if enabled is false, the whitelisting feature is removed.
        # This is like removing the check that checks if an API is whitelisted.
        # This is equivalent to continuing with the usual access control checks, and removing all the code that implements whitelisting.
        # if enabled is true, then all users except SuperAdmin can access only the APIs in requests
        # SuperAdmin can access all APIs.
        # SuperAdmin is defined by the SuperAdmin certificate, which is configured in the opensearch.yml setting: plugins.security.authcz.admin_dn:
        # Refer to the example setting in opensearch.yml.example, and the opendistro documentation to know more about configuring SuperAdmin.
        #
        # requests - map of whitelisted endpoints, and the whitelisted HTTP requests for those endpoints

        # Examples showing how to configure this yml file (make sure the _meta data from above is also there):
        # Example 1:
        # To enable whitelisting and whitelist GET /_cluster/settings
        #
        #config:
        #  enabled: true
        #  requests:
        #    /_cluster/settings:
        #      - GET
        #
        # Example 2:
        # If you want to whitelist multiple request methods for /_cluster/settings (GET,PUT):
        #
        #config:
        #  enabled: true
        #  requests:
        #    /_cluster/settings:
        #      - GET
        #      - PUT
        #
        # Example 3:
        # If you want to whitelist other APIs as well, for example GET /_cat/nodes, and GET /_cat/shards:
        #
        #config:
        #  enabled: true
        #  requests:
        #    /_cluster/settings:
        #      - GET
        #      - PUT
        #    /_cat/nodes:
        #      - GET
        #    /_cat/shards:
        #      - GET
        #
        # Example 4:
        # If you want to disable the whitelisting feature, set enabled to false.
        #  enabled: false
        #  requests:
        #    /_cluster/settings:
        #      - GET
        #
        #At this point, all APIs become whitelisted because the feature to whitelist is off, so requests is irrelevant.


        #this name must be config
        config:
          enabled: false
          requests:
            /_cluster/settings:
              - GET
            /_cat/nodes:
              - GET


# How long to wait for opensearch to stop gracefully
terminationGracePeriod: 120

sysctlVmMaxMapCount: 262144

readinessProbe:
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 3
  timeoutSeconds: 2000

## Use an alternate scheduler.
## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
##
schedulerName: ""

imagePullSecrets:
  - name: ommited
nodeSelector: {}
tolerations: []

# Enabling this will publically expose your OpenSearch instance.
# Only enable this if you have security enabled on your cluster
ingress:
  enabled: true
  annotations:
    kubernetes.io/ingress.class: traefik
    # kubernetes.io/tls-acme: "true"
  path: /
  hosts:
    - ommited
  # tls:
  #  - secretName: ommited
  #    hosts:
  #      - ommited

nameOverride: ""
fullnameOverride: ""

masterTerminationFix: false

lifecycle: {}
  # preStop:
  #   exec:
  #     command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"]
  # postStart:
  #   exec:
  #     command:
  #       - bash
  #       - -c
  #       - |
  #         #!/bin/bash
  #         # Add a template to adjust number of shards/replicas1
  #         TEMPLATE_NAME=my_template
  #         INDEX_PATTERN="logstash-*"
  #         SHARD_COUNT=8
  #         REPLICA_COUNT=1
  #         ES_URL=http://localhost:9200
  #         while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
  #         curl -XPUT "$ES_URL/_template/$TEMPLATE_NAME" -H 'Content-Type: application/json' -d'{"index_patterns":['\""$INDEX_PATTERN"\"'],"settings":{"number_of_shards":'$SHARD_COUNT',"number_of_replicas":'$REPLICA_COUNT'}}'

keystore: []

networkPolicy:
  ## Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now.
  ## In order for a Pod to access OpenSearch, it needs to have the following label:
  ## {{ template "uname" . }}-client: "true"
  ## Example for default configuration to access HTTP port:
  ## opensearch-master-http-client: "true"
  ## Example for default configuration to access transport port:
  ## opensearch-master-transport-client: "true"

  http:
    enabled: false

# Deprecated
# please use the above podSecurityContext.fsGroup instead
fsGroup: ""

## Set optimal sysctl's. This requires privilege. Can be disabled if
## the system has already been preconfigured. (Ex: https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html)
## Also see: https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/
sysctl:
  enabled: false