ebuildy/elasticsearch.log

## elasticsearch.log
{"type": "server", "timestamp": "2021-07-07T07:33:02,722Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "version[7.13.2], pid[6], build[default/docker/4d960a0733be83dd2543ca018aa4ddc42e956800/2021-06-10T21:01:55.251515791Z], OS[Linux/4.15.0-121-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/16/16+36]" }
{"type": "server", "timestamp": "2021-07-07T07:33:02,725Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2021-07-07T07:33:02,725Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-6835614695399182291, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xmx2g, -Xms2g, -XX:MaxDirectMemorySize=1073741824, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,429Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,429Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,430Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,430Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,430Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,430Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,430Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,431Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,431Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,431Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,431Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,432Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,432Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,432Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,432Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,433Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,433Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,433Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,433Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,433Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,434Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,434Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,434Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,434Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,435Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,435Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,435Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,435Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,435Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,436Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,436Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,436Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,436Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,436Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,436Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,437Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,437Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,437Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,437Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,437Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,438Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,438Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,438Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,438Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,438Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,439Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,439Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,439Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,439Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,439Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,439Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,440Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,440Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,440Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,440Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,440Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,441Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,441Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "loaded plugin [repository-s3]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,458Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-07-07T07:33:06,474Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/rbd4)]], net usable_space [13.5gb], net total_space [19.5gb], types [ext4]" }
{"type": "server", "timestamp": "2021-07-07T07:33:06,475Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "heap size [2gb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,559Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,565Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[node.remote_cluster_client] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,566Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[node.ingest] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,567Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[node.ml] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-07-07T07:33:06,716Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "node name [djobi-metrics-es-0], node ID [JA04wYufQcaW2fJPk-gCvg], cluster name [djobi-metrics], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,718Z", "level": "DEPRECATION", "component": "o.e.d.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "legacy role settings [node.data, node.remote_cluster_client, node.ingest, node.master, node.ml] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:06,723Z", "level": "DEPRECATION", "component": "o.e.d.c.u.c.EsExecutors", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "setting [node.processors] to value [4] which is more than available processors [1] is deprecated" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-07T07:33:10,490Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[xpack.monitoring.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-07-07T07:33:13,413Z", "level": "INFO", "component": "o.e.i.g.LocalDatabases", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2021-07-07T07:33:13,415Z", "level": "INFO", "component": "o.e.i.g.DatabaseRegistry", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-6835614695399182291/geoip-databases/JA04wYufQcaW2fJPk-gCvg]" }
{"type": "server", "timestamp": "2021-07-07T07:33:14,027Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=1mb, suggested_max_allocation_size=1mb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=4mb}]" }
{"type": "server", "timestamp": "2021-07-07T07:33:14,129Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2021-07-07T07:33:14,573Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2021-07-07T07:33:14,961Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "initialized" }
{"type": "server", "timestamp": "2021-07-07T07:33:14,962Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "starting ..." }
{"type": "server", "timestamp": "2021-07-07T07:33:15,188Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2021-07-07T07:33:15,314Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "publish_address {10.236.109.148:9300}, bound_addresses {0.0.0.0:9300}" }
{"type": "server", "timestamp": "2021-07-07T07:33:19,344Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2021-07-07T07:33:19,346Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "cluster UUID [8xgFHc_kRUiul-gv4SsHyg]" }
{"type": "server", "timestamp": "2021-07-07T07:33:20,578Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "master node changed {previous [], current [{djobi-metrics-es-2}{mKWETQsHSlqVZr6p1RKnFw}{Z8mHXQs0SzGkeKpx4F88UQ}{10.236.79.251}{10.236.79.251:9300}{cdfhilmrstw}]}, added {{djobi-metrics-es-1}{66geZ-jpT-K2efaZ56O48Q}{8Z8I6qhFSaOlZBpdo7TdHw}{10.236.72.177}{10.236.72.177:9300}{cdfhilmrstw}, {djobi-metrics-es-2}{mKWETQsHSlqVZr6p1RKnFw}{Z8mHXQs0SzGkeKpx4F88UQ}{10.236.79.251}{10.236.79.251:9300}{cdfhilmrstw}}, term: 54, version: 7737, reason: ApplyCommitRequest{term=54, version=7737, sourceNode={djobi-metrics-es-2}{mKWETQsHSlqVZr6p1RKnFw}{Z8mHXQs0SzGkeKpx4F88UQ}{10.236.79.251}{10.236.79.251:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}}" }
{"type": "server", "timestamp": "2021-07-07T07:33:21,865Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "license [61f2d5b0-7a30-40ca-ad5a-4e06e00baa8b] mode [basic] - valid" }
{"type": "server", "timestamp": "2021-07-07T07:33:21,978Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "publish_address {10.236.109.148:9200}, bound_addresses {0.0.0.0:9200}", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-07T07:33:21,978Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "started", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-07T07:33:44,272Z", "level": "INFO", "component": "o.e.m.j.JvmGcMonitorService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[gc][29] overhead, spent [250ms] collecting in the last [1s]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T02:02:56,174Z", "level": "WARN", "component": "o.e.m.j.JvmGcMonitorService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[gc][66564] overhead, spent [677ms] collecting in the last [1s]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "deprecation.elasticsearch", "timestamp": "2021-07-08T04:03:32,688Z", "level": "DEPRECATION", "component": "o.e.d.r.a.d.RestUpdateAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[types removal] Specifying types in document update requests is deprecated, use the endpoint /{index}/_update/{id} instead.", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:30:46,691Z", "level": "WARN", "component": "o.e.c.NodeConnectionsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "failed to connect to {djobi-metrics-es-1}{66geZ-jpT-K2efaZ56O48Q}{8Z8I6qhFSaOlZBpdo7TdHw}{10.236.72.177}{10.236.72.177:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true} (tried [1] times)", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [djobi-metrics-es-1][10.236.72.177:9300] connect_exception",
"at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:971) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:277) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:31) ~[elasticsearch-core-7.13.2.jar:7.13.2]",
"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
"at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2158) ~[?:?]",
"at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:46) ~[elasticsearch-core-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:57) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.236.72.177/10.236.72.177:9300",
"Caused by: java.net.ConnectException: Connection refused",
"at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
"at sun.nio.ch.Net.pollConnectNow(Net.java:669) ~[?:?]",
"at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:944) ~[?:?]",
"at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
"... 7 more"] }
{"type": "server", "timestamp": "2021-07-08T12:31:46,718Z", "level": "WARN", "component": "o.e.c.NodeConnectionsService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "failed to connect to {djobi-metrics-es-1}{66geZ-jpT-K2efaZ56O48Q}{8Z8I6qhFSaOlZBpdo7TdHw}{10.236.72.177}{10.236.72.177:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true} (tried [7] times)", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [djobi-metrics-es-1][10.236.72.177:9300] connect_exception",
"at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:971) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:277) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:31) ~[elasticsearch-core-7.13.2.jar:7.13.2]",
"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
"at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2158) ~[?:?]",
"at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:46) ~[elasticsearch-core-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:57) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.236.72.177/10.236.72.177:9300",
"Caused by: java.net.ConnectException: Connection refused",
"at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
"at sun.nio.ch.Net.pollConnectNow(Net.java:669) ~[?:?]",
"at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:944) ~[?:?]",
"at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
"... 7 more"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,398Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "stopping ...", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:07,403Z", "level": "INFO", "component": "o.e.x.w.WatcherService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "stopping watch service, reason [shutdown initiated]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:07,404Z", "level": "INFO", "component": "o.e.x.w.WatcherLifeCycleService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "watcher has stopped and shutdown", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:07,809Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "path: /_license, params: {human=false}", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.node.NodeClosedException: node closed {djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onClusterServiceClose(TransportMasterNodeAction.java:212) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onClusterServiceClose(ClusterStateObserver.java:317) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onClose(ClusterStateObserver.java:226) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.service.ClusterApplierService.addTimeoutListener(ClusterApplierService.java:252) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:165) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:203) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retryOnMasterChange(TransportMasterNodeAction.java:189) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$200(TransportMasterNodeAction.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:175) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,814Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:47) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onClusterServiceClose(TransportMasterNodeAction.java:212) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onClusterServiceClose(ClusterStateObserver.java:317) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onClose(ClusterStateObserver.java:226) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.service.ClusterApplierService.addTimeoutListener(ClusterApplierService.java:252) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:165) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:203) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retryOnMasterChange(TransportMasterNodeAction.java:189) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$200(TransportMasterNodeAction.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:175) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,825Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:445) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.newResponseAsync(TransportNodesAction.java:157) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.newResponse(TransportNodesAction.java:133) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.lambda$finishHim$0(TransportNodesAction.java:267) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,848Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.sendMessage(Netty4TcpChannel.java:135) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:131) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:122) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendErrorResponse(OutboundHandler.java:116) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:397) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,848Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,848Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,855Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.sendMessage(Netty4TcpChannel.java:135) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:131) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:122) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendErrorResponse(OutboundHandler.java:116) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:397) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:07,848Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.sendMessage(Netty4TcpChannel.java:135) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:131) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:122) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendErrorResponse(OutboundHandler.java:116) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:397) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,053Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "path: /.kibana_7.13.2/_doc/config%3A7.13.2, params: {index=.kibana_7.13.2, id=config:7.13.2}", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.SendRequestTransportException: [djobi-metrics-es-0][10.236.109.148:9300][indices:data/read/get[s]]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:858) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:670) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:230) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.onFailure(TransportSingleShardAction.java:194) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.access$1100(TransportSingleShardAction.java:125) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction$2.handleException(TransportSingleShardAction.java:245) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.node.NodeClosedException: node closed {djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:839) ~[elasticsearch-7.13.2.jar:7.13.2]",
"... 13 more"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,055Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:47) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:213) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.onFailure(TransportSingleShardAction.java:194) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.access$1100(TransportSingleShardAction.java:125) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction$2.handleException(TransportSingleShardAction.java:245) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:670) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:230) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.onFailure(TransportSingleShardAction.java:194) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.access$1100(TransportSingleShardAction.java:125) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction$2.handleException(TransportSingleShardAction.java:245) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,068Z", "level": "WARN", "component": "o.e.t.OutboundHandler", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "send message failed [channel: Netty4TcpChannel{localAddress=/10.236.109.148:9300, remoteAddress=/10.236.79.251:34294, profile=default}]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.TransportException: Cannot send message, event loop is shutting down.",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.sendMessage(Netty4TcpChannel.java:138) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:131) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:122) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendErrorResponse(OutboundHandler.java:116) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:397) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,068Z", "level": "WARN", "component": "o.e.t.OutboundHandler", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "send message failed [channel: Netty4TcpChannel{localAddress=/10.236.109.148:9300, remoteAddress=/10.236.79.251:34296, profile=default}]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.TransportException: Cannot send message, event loop is shutting down.",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.sendMessage(Netty4TcpChannel.java:138) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:131) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:122) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendErrorResponse(OutboundHandler.java:116) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:397) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,068Z", "level": "WARN", "component": "o.e.t.OutboundHandler", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "send message failed [channel: Netty4TcpChannel{localAddress=/10.236.109.148:9300, remoteAddress=/10.236.79.251:34292, profile=default}]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.TransportException: Cannot send message, event loop is shutting down.",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.sendMessage(Netty4TcpChannel.java:138) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:131) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:122) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.OutboundHandler.sendErrorResponse(OutboundHandler.java:116) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onFailure(ActionListener.java:397) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,080Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "path: /.kibana_task_manager/_update_by_query, params: {ignore_unavailable=true, refresh=true, conflicts=proceed, index=.kibana_task_manager}", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:661) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:694) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:810) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:802) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:135) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:71) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:300) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:337) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.node.NodeClosedException: node closed {djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:839) ~[elasticsearch-7.13.2.jar:7.13.2]",
"... 20 more"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,083Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:47) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.lambda$finishHim$3(AbstractAsyncBulkByScrollAction.java:518) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.cleanup(ClientScrollableHitSource.java:120) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:110) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:498) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:142) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$1.onFailure(ClientScrollableHitSource.java:89) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunAfterActionListener.onFailure(ActionListener.java:348) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.raisePhaseFailure(AbstractSearchAsyncAction.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:661) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:694) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:810) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:802) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:135) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:71) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:300) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:337) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,088Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[[.monitoring-beats-7-mb-2021.07.08][0]] failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,091Z", "level": "WARN", "component": "o.e.c.a.s.ShardStateAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "node closed while execution action [internal:cluster/shard/failure] for shard entry [shard id [[.monitoring-beats-7-mb-2021.07.08][0]], allocation id [xpzCu6pVQ8GkDfW8qMtsIA], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]], failure [NodeDisconnectedException[[djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,091Z", "level": "WARN", "component": "o.e.t.TransportService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Transport response handler not found of id [607051]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,088Z", "level": "WARN", "component": "o.e.a.s.TransportClearScrollAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Clear SC failed on node[{djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.SendRequestTransportException: [djobi-metrics-es-0][10.236.109.148:9300][indices:data/read/search[free_context/scroll]]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:858) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendFreeContext(SearchTransportService.java:106) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$4(ClearScrollController.java:110) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchScrollAsyncAction.collectNodesAndRun(SearchScrollAsyncAction.java:104) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.cleanScrollIds(ClearScrollController.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$new$1(ClearScrollController.java:67) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.run(ClearScrollController.java:76) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:39) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:20) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.clearScroll(AbstractClient.java:629) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.clearScroll(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.refreshAndFinish(AbstractAsyncBulkByScrollAction.java:472) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.onBulkResponse(AbstractAsyncBulkByScrollAction.java:420) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:369) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:366) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:159) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onClusterServiceClose(TransportReplicationAction.java:838) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onClusterServiceClose(ClusterStateObserver.java:317) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onClose(ClusterStateObserver.java:226) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.service.ClusterApplierService.addTimeoutListener(ClusterApplierService.java:252) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:165) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.node.NodeClosedException: node closed {djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:839) ~[elasticsearch-7.13.2.jar:7.13.2]",
"... 51 more"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,088Z", "level": "WARN", "component": "o.e.a.s.TransportClearScrollAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Clear SC failed on node[{djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.SendRequestTransportException: [djobi-metrics-es-0][10.236.109.148:9300][indices:data/read/search[free_context/scroll]]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:858) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendFreeContext(SearchTransportService.java:106) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$4(ClearScrollController.java:110) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchScrollAsyncAction.collectNodesAndRun(SearchScrollAsyncAction.java:104) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.cleanScrollIds(ClearScrollController.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$new$1(ClearScrollController.java:67) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.run(ClearScrollController.java:76) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:39) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:20) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.clearScroll(AbstractClient.java:629) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.clearScroll(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.refreshAndFinish(AbstractAsyncBulkByScrollAction.java:472) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.onBulkResponse(AbstractAsyncBulkByScrollAction.java:420) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:369) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:366) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:159) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onClusterServiceClose(TransportReplicationAction.java:838) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onClusterServiceClose(ClusterStateObserver.java:317) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onClose(ClusterStateObserver.java:226) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.service.ClusterApplierService.addTimeoutListener(ClusterApplierService.java:252) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:165) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.node.NodeClosedException: node closed {djobi-metrics-es-0}{JA04wYufQcaW2fJPk-gCvg}{3DmYljWmRVisoFRi4UmmZA}{10.236.109.148}{10.236.109.148:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:839) ~[elasticsearch-7.13.2.jar:7.13.2]",
"... 51 more"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,088Z", "level": "WARN", "component": "o.e.a.s.TransportClearScrollAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Clear SC failed on node[{djobi-metrics-es-2}{mKWETQsHSlqVZr6p1RKnFw}{Z8mHXQs0SzGkeKpx4F88UQ}{10.236.79.251}{10.236.79.251:9300}{cdfhilmrstw}{xpack.installed=true, transform.node=true}]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.NodeNotConnectedException: [djobi-metrics-es-2][10.236.79.251:9300] Node not connected",
"at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:178) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:780) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.getConnection(SearchTransportService.java:370) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$4(ClearScrollController.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchScrollAsyncAction.collectNodesAndRun(SearchScrollAsyncAction.java:104) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.cleanScrollIds(ClearScrollController.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$new$1(ClearScrollController.java:67) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.run(ClearScrollController.java:76) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:39) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:20) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.clearScroll(AbstractClient.java:629) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.clearScroll(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.refreshAndFinish(AbstractAsyncBulkByScrollAction.java:472) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.onBulkResponse(AbstractAsyncBulkByScrollAction.java:420) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:369) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:366) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:159) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,092Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "path: /.kibana_7.13.2/_search, params: {rest_total_hits_as_int=true, size=20, index=.kibana_7.13.2, from=0}", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:661) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:694) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:768) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:810) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:802) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:135) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:71) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:300) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:337) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.tasks.TaskCancelledException: The parent task was cancelled, shouldn't start any child tasks",
"at org.elasticsearch.tasks.TaskManager$CancellableTaskHolder.registerChildConnection(TaskManager.java:573) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.tasks.TaskManager.registerChildConnection(TaskManager.java:211) ~[elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:727) ~[elasticsearch-7.13.2.jar:7.13.2]",
"... 19 more"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,094Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:47) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestCancellableNodeClient$1.onFailure(RestCancellableNodeClient.java:96) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunAfterActionListener.onFailure(ActionListener.java:348) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.raisePhaseFailure(AbstractSearchAsyncAction.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:661) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:694) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:768) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:810) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:802) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendExecuteQuery(SearchTransportService.java:135) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction.executePhaseOnShard(SearchQueryThenFetchAsyncAction.java:71) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:300) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:337) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:316) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,097Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "path: /_license, params: {human=false}", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[djobi-metrics-es-2][10.236.79.251:9300][cluster:monitor/xpack/license/get] disconnected]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:197) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retryOnMasterChange(TransportMasterNodeAction.java:189) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$200(TransportMasterNodeAction.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:175) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.transport.NodeDisconnectedException: [djobi-metrics-es-2][10.236.79.251:9300][cluster:monitor/xpack/license/get] disconnected"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,098Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onFailure(RestActionListener.java:47) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:197) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retryOnMasterChange(TransportMasterNodeAction.java:189) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$200(TransportMasterNodeAction.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:175) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$8.run(TransportService.java:1145) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,099Z", "level": "WARN", "component": "o.e.t.TaskCancellationService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Cannot send ban for tasks with the parent [JA04wYufQcaW2fJPk-gCvg:1422223] for connection [org.elasticsearch.transport.TcpTransport$NodeChannels@3c3b2f57]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,100Z", "level": "INFO", "component": "o.e.t.TaskCancellationService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "failed to remove the parent ban for task JA04wYufQcaW2fJPk-gCvg:1422223 for connection org.elasticsearch.transport.TcpTransport$NodeChannels@3c3b2f57", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,101Z", "level": "WARN", "component": "o.e.t.TransportService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Transport response handler not found of id [607119]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,101Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[[.monitoring-beats-7-mb-2021.07.08][0]] failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,102Z", "level": "WARN", "component": "o.e.c.a.s.ShardStateAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "node closed while execution action [internal:cluster/shard/failure] for shard entry [shard id [[.monitoring-beats-7-mb-2021.07.08][0]], allocation id [xpzCu6pVQ8GkDfW8qMtsIA], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]], failure [NodeDisconnectedException[[djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,102Z", "level": "WARN", "component": "o.e.t.TransportService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Transport response handler not found of id [607057]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,100Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:445) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.newResponseAsync(TransportNodesAction.java:157) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.newResponse(TransportNodesAction.java:133) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.lambda$finishHim$0(TransportNodesAction.java:267) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,100Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:445) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.newResponseAsync(TransportNodesAction.java:157) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.newResponse(TransportNodesAction.java:133) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.lambda$finishHim$0(TransportNodesAction.java:267) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,102Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[[.monitoring-beats-7-mb-2021.07.08][0]] failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,103Z", "level": "WARN", "component": "o.e.c.a.s.ShardStateAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "node closed while execution action [internal:cluster/shard/failure] for shard entry [shard id [[.monitoring-beats-7-mb-2021.07.08][0]], allocation id [xpzCu6pVQ8GkDfW8qMtsIA], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]], failure [NodeDisconnectedException[[djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,103Z", "level": "WARN", "component": "o.e.t.TransportService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Transport response handler not found of id [607087]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,103Z", "level": "WARN", "component": "o.e.a.b.TransportShardBulkAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "[[.monitoring-beats-7-mb-2021.07.08][0]] failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,104Z", "level": "WARN", "component": "o.e.c.a.s.ShardStateAction", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "node closed while execution action [internal:cluster/shard/failure] for shard entry [shard id [[.monitoring-beats-7-mb-2021.07.08][0]], allocation id [xpzCu6pVQ8GkDfW8qMtsIA], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [.monitoring-beats-7-mb-2021.07.08][0], node[mKWETQsHSlqVZr6p1RKnFw], [R], s[STARTED], a[id=xpzCu6pVQ8GkDfW8qMtsIA]], failure [NodeDisconnectedException[[djobi-metrics-es-2][10.236.79.251:9300][indices:data/write/bulk[s][r]] disconnected]], markAsStale [true]]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,104Z", "level": "WARN", "component": "o.e.t.TransportService", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Transport response handler not found of id [607089]", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg"  }
{"type": "server", "timestamp": "2021-07-08T12:32:08,100Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.lambda$finishHim$3(AbstractAsyncBulkByScrollAction.java:516) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.cleanup(ClientScrollableHitSource.java:120) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.lambda$close$2(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$2.onResponse(ClientScrollableHitSource.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$2.onResponse(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.onFailedFreedContext(ClearScrollController.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$3(ClearScrollController.java:111) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:142) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendFreeContext(SearchTransportService.java:106) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$4(ClearScrollController.java:110) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchScrollAsyncAction.collectNodesAndRun(SearchScrollAsyncAction.java:104) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.cleanScrollIds(ClearScrollController.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$new$1(ClearScrollController.java:67) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.run(ClearScrollController.java:76) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:39) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:20) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.clearScroll(AbstractClient.java:629) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.clearScroll(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.refreshAndFinish(AbstractAsyncBulkByScrollAction.java:472) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.onBulkResponse(AbstractAsyncBulkByScrollAction.java:420) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:369) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:366) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:159) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onClusterServiceClose(TransportReplicationAction.java:838) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onClusterServiceClose(ClusterStateObserver.java:317) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onClose(ClusterStateObserver.java:226) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.service.ClusterApplierService.addTimeoutListener(ClusterApplierService.java:252) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:165) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,100Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.lambda$finishHim$3(AbstractAsyncBulkByScrollAction.java:516) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.cleanup(ClientScrollableHitSource.java:120) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.lambda$close$2(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$2.onResponse(ClientScrollableHitSource.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$2.onResponse(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.onFailedFreedContext(ClearScrollController.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$3(ClearScrollController.java:111) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:142) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchTransportService.sendFreeContext(SearchTransportService.java:106) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$4(ClearScrollController.java:110) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchScrollAsyncAction.collectNodesAndRun(SearchScrollAsyncAction.java:104) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.cleanScrollIds(ClearScrollController.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$new$1(ClearScrollController.java:67) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.run(ClearScrollController.java:76) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:39) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:20) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.clearScroll(AbstractClient.java:629) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.clearScroll(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.refreshAndFinish(AbstractAsyncBulkByScrollAction.java:472) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.onBulkResponse(AbstractAsyncBulkByScrollAction.java:420) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:369) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:366) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:159) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onClusterServiceClose(TransportReplicationAction.java:838) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onClusterServiceClose(ClusterStateObserver.java:317) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onClose(ClusterStateObserver.java:226) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.service.ClusterApplierService.addTimeoutListener(ClusterApplierService.java:252) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:165) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-07-08T12:32:08,100Z", "level": "ERROR", "component": "i.n.u.c.D.rejectedExecution", "cluster.name": "djobi-metrics", "node.name": "djobi-metrics-es-0", "message": "Failed to submit a listener notification task. Event loop shut down?", "cluster.uuid": "8xgFHc_kRUiul-gv4SsHyg", "node.id": "JA04wYufQcaW2fJPk-gCvg" ,
"stacktrace": ["java.util.concurrent.RejectedExecutionException: event executor terminated",
"at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) ~[netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:498) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:109) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPromise.setFailure(DefaultChannelPromise.java:89) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:994) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:796) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:758) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1020) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:299) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpChannel.sendResponse(Netty4HttpChannel.java:32) [transport-netty4-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.http.DefaultRestChannel.sendResponse(DefaultRestChannel.java:127) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.RestController$ResourceHandlingHttpChannel.sendResponse(RestController.java:516) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestResponseListener.processResponse(RestResponseListener.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.rest.action.RestActionListener.onResponse(RestActionListener.java:36) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.lambda$finishHim$3(AbstractAsyncBulkByScrollAction.java:516) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.cleanup(ClientScrollableHitSource.java:120) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.lambda$close$2(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$2.onResponse(ClientScrollableHitSource.java:107) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource$2.onResponse(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.onFailedFreedContext(ClearScrollController.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$cleanScrollIds$4(ClearScrollController.java:113) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.SearchScrollAsyncAction.collectNodesAndRun(SearchScrollAsyncAction.java:104) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.cleanScrollIds(ClearScrollController.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.lambda$new$1(ClearScrollController.java:67) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.ClearScrollController.run(ClearScrollController.java:76) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:39) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.search.TransportClearScrollAction.doExecute(TransportClearScrollAction.java:20) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.client.support.AbstractClient.clearScroll(AbstractClient.java:629) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ClientScrollableHitSource.clearScroll(ClientScrollableHitSource.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.ScrollableHitSource.close(ScrollableHitSource.java:108) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.finishHim(AbstractAsyncBulkByScrollAction.java:511) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.refreshAndFinish(AbstractAsyncBulkByScrollAction.java:472) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.onBulkResponse(AbstractAsyncBulkByScrollAction.java:420) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:369) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction$2.onResponse(AbstractAsyncBulkByScrollAction.java:366) [reindex-client-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.finishHim(Retry.java:159) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:103) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.Retry$RetryHandler.onResponse(Retry.java:66) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.ActionListener$RunBeforeActionListener.onResponse(ActionListener.java:387) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:538) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:533) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:853) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:825) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:880) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:860) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:759) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:685) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:787) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performLocalAction(TransportReplicationAction.java:758) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:745) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:844) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:324) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:138) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:109) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:101) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:830) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$1.handleException(TransportReplicationAction.java:809) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$5.handleException(TransportService.java:738) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1283) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:312) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.2.jar:7.13.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.2.jar:7.13.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }