Skip to content

Instantly share code, notes, and snippets.

@fschoell
Last active December 14, 2021 11:51
Show Gist options
  • Save fschoell/9ec4914893f13b2950cd48b0188a24cc to your computer and use it in GitHub Desktop.
Save fschoell/9ec4914893f13b2950cd48b0188a24cc to your computer and use it in GitHub Desktop.
Elasticsearch
GET _cluster/stats
{
"_nodes" : {
"total" : 19,
"successful" : 19,
"failed" : 0
},
"cluster_name" : "esh1",
"cluster_uuid" : "BboZSFuJQVqY2EUVAHDNQQ",
"timestamp" : 1639482654342,
"status" : "green",
"indices" : {
"count" : 101,
"shards" : {
"total" : 496,
"primaries" : 227,
"replication" : 1.1850220264317182,
"index" : {
"shards" : {
"min" : 2,
"max" : 24,
"avg" : 4.910891089108911
},
"primaries" : {
"min" : 1,
"max" : 12,
"avg" : 2.2475247524752477
},
"replication" : {
"min" : 1.0,
"max" : 5.0,
"avg" : 1.1782178217821782
}
}
},
"docs" : {
"count" : 966479420,
"deleted" : 11775873
},
"store" : {
"size_in_bytes" : 613785830069,
"total_data_set_size_in_bytes" : 613785830069,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 11822984,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 70006049,
"total_count" : 6330734,
"hit_count" : 41098,
"miss_count" : 6289636,
"cache_size" : 10010,
"cache_count" : 11109,
"evictions" : 1099
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 6026,
"memory_in_bytes" : 55522264,
"terms_memory_in_bytes" : 39986384,
"stored_fields_memory_in_bytes" : 3767008,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 74880,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 11693992,
"index_writer_memory_in_bytes" : 103132268,
"version_map_memory_in_bytes" : 5033352,
"fixed_bit_set_memory_in_bytes" : 128185176,
"max_unsafe_auto_id_timestamp" : 1639482427822,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "alias",
"count" : 91,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "binary",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "boolean",
"count" : 1081,
"index_count" : 30,
"script_count" : 0
},
{
"name" : "byte",
"count" : 8,
"index_count" : 8,
"script_count" : 0
},
{
"name" : "constant_keyword",
"count" : 26,
"index_count" : 9,
"script_count" : 0
},
{
"name" : "date",
"count" : 1637,
"index_count" : 72,
"script_count" : 0
},
{
"name" : "date_nanos",
"count" : 11,
"index_count" : 9,
"script_count" : 0
},
{
"name" : "date_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "double",
"count" : 260,
"index_count" : 8,
"script_count" : 0
},
{
"name" : "double_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "flattened",
"count" : 297,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "float",
"count" : 319,
"index_count" : 27,
"script_count" : 0
},
{
"name" : "float_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "geo_point",
"count" : 78,
"index_count" : 8,
"script_count" : 0
},
{
"name" : "geo_shape",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "half_float",
"count" : 48,
"index_count" : 12,
"script_count" : 0
},
{
"name" : "integer",
"count" : 215,
"index_count" : 20,
"script_count" : 0
},
{
"name" : "integer_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "ip",
"count" : 960,
"index_count" : 22,
"script_count" : 0
},
{
"name" : "ip_range",
"count" : 8,
"index_count" : 8,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 33058,
"index_count" : 73,
"script_count" : 0
},
{
"name" : "long",
"count" : 8923,
"index_count" : 58,
"script_count" : 0
},
{
"name" : "long_range",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "match_only_text",
"count" : 195,
"index_count" : 3,
"script_count" : 0
},
{
"name" : "nested",
"count" : 157,
"index_count" : 38,
"script_count" : 0
},
{
"name" : "object",
"count" : 7413,
"index_count" : 59,
"script_count" : 0
},
{
"name" : "scaled_float",
"count" : 7,
"index_count" : 7,
"script_count" : 0
},
{
"name" : "shape",
"count" : 1,
"index_count" : 1,
"script_count" : 0
},
{
"name" : "short",
"count" : 722,
"index_count" : 8,
"script_count" : 0
},
{
"name" : "text",
"count" : 794,
"index_count" : 54,
"script_count" : 0
},
{
"name" : "version",
"count" : 4,
"index_count" : 4,
"script_count" : 0
},
{
"name" : "wildcard",
"count" : 58,
"index_count" : 7,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [ ],
"analyzer_types" : [ ],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [ ],
"built_in_analyzers" : [ ]
},
"versions" : [
{
"version" : "7.9.3",
"index_count" : 9,
"primary_shard_count" : 9,
"total_primary_bytes" : 3856517987
},
{
"version" : "7.10.2",
"index_count" : 3,
"primary_shard_count" : 3,
"total_primary_bytes" : 11349236
},
{
"version" : "7.11.1",
"index_count" : 11,
"primary_shard_count" : 26,
"total_primary_bytes" : 12085958743
},
{
"version" : "7.13.1",
"index_count" : 17,
"primary_shard_count" : 33,
"total_primary_bytes" : 24874975964
},
{
"version" : "7.14.1",
"index_count" : 14,
"primary_shard_count" : 19,
"total_primary_bytes" : 10736861466
},
{
"version" : "7.15.1",
"index_count" : 16,
"primary_shard_count" : 24,
"total_primary_bytes" : 12745956680
},
{
"version" : "7.15.2",
"index_count" : 25,
"primary_shard_count" : 96,
"total_primary_bytes" : 198437358448
},
{
"version" : "7.16.1",
"index_count" : 6,
"primary_shard_count" : 17,
"total_primary_bytes" : 21004788303
}
]
},
"nodes" : {
"count" : {
"total" : 19,
"coordinating_only" : 0,
"data" : 12,
"data_cold" : 12,
"data_content" : 12,
"data_frozen" : 12,
"data_hot" : 12,
"data_warm" : 12,
"ingest" : 4,
"master" : 3,
"ml" : 0,
"remote_cluster_client" : 0,
"transform" : 12,
"voting_only" : 0
},
"versions" : [
"7.16.1"
],
"os" : {
"available_processors" : 1376,
"allocated_processors" : 1376,
"names" : [
{
"name" : "Linux",
"count" : 19
}
],
"pretty_names" : [
{
"pretty_name" : "Debian GNU/Linux 11 (bullseye)",
"count" : 19
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 19
}
],
"mem" : {
"total_in_bytes" : 25141624598528,
"free_in_bytes" : 1274116526080,
"used_in_bytes" : 23867508072448,
"free_percent" : 5,
"used_percent" : 95
}
},
"process" : {
"cpu" : {
"percent" : 13
},
"open_file_descriptors" : {
"min" : 745,
"max" : 1430,
"avg" : 1191
}
},
"jvm" : {
"max_uptime_in_millis" : 75958328,
"versions" : [
{
"version" : "17.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "17.0.1+12",
"vm_vendor" : "Eclipse Adoptium",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 19
}
],
"mem" : {
"heap_used_in_bytes" : 11036731888,
"heap_max_in_bytes" : 20401094656
},
"threads" : 5097
},
"fs" : {
"total_in_bytes" : 115451465449472,
"free_in_bytes" : 114009401143296,
"available_in_bytes" : 113311592706048
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 19
},
"http_types" : {
"security4" : 19
}
},
"discovery_types" : {
"zen" : 19
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "deb",
"count" : 19
}
],
"ingest" : {
"number_of_pipelines" : 31,
"processor_stats" : {
"append" : {
"count" : 185080012,
"failed" : 0,
"current" : 0,
"time_in_millis" : 156622
},
"conditional" : {
"count" : 369751631,
"failed" : 0,
"current" : 0,
"time_in_millis" : 4801940
},
"convert" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"date" : {
"count" : 92403874,
"failed" : 0,
"current" : 0,
"time_in_millis" : 2560825
},
"geoip" : {
"count" : 184807748,
"failed" : 0,
"current" : 0,
"time_in_millis" : 6409259
},
"grok" : {
"count" : 277347754,
"failed" : 0,
"current" : 0,
"time_in_millis" : 8956547
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"remove" : {
"count" : 184943880,
"failed" : 0,
"current" : 0,
"time_in_millis" : 410354
},
"rename" : {
"count" : 369887760,
"failed" : 0,
"current" : 0,
"time_in_millis" : 1011416
},
"script" : {
"count" : 92403874,
"failed" : 0,
"current" : 0,
"time_in_millis" : 3768731
},
"set" : {
"count" : 185080012,
"failed" : 0,
"current" : 0,
"time_in_millis" : 2456490
},
"set_security_user" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"split" : {
"count" : 184807748,
"failed" : 0,
"current" : 0,
"time_in_millis" : 2271708
},
"uri_parts" : {
"count" : 92403874,
"failed" : 895,
"current" : 0,
"time_in_millis" : 251978
},
"user_agent" : {
"count" : 92403874,
"failed" : 0,
"current" : 0,
"time_in_millis" : 2573237
}
}
}
}
}
GET _cat/health?v
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1639482090 11:41:30 esh1 green 19 12 496 227 2 0 0 0 - 100.0%
GET _nodes/hot_threads?threads=9999
::: {esh1-data12c}{s6GsYXLhR4u3M1L0cQFhMw}{jCzY9DNPQPCjt3vO6mOvZw}{172.22.3.189}{172.22.3.189:9300}{cdfhstw}{rack_id=dom12, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.051Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (34.8micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data12d}{aA0yEBu6S2aJ-h23kQugLw}{KwuOd1XiSF-nSAM874_OLw}{172.22.3.162}{172.22.3.162:9300}{cdfhstw}{rack_id=dom12, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.051Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (40.5micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data11c}{0G1vhbepQLScTRZa6e7Pzw}{sbzbtfZmRXyV6VcizFXaCA}{172.22.3.187}{172.22.3.187:9300}{cdfhstw}{rack_id=dom11, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.048Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (38.7micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data11b}{u-MegzNJTAC3iUZj9E3CNg}{YNaRJ3CaRi2h8DsoqojYQA}{172.22.3.182}{172.22.3.182:9300}{cdfhstw}{rack_id=dom11, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.048Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (44.5micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-master3}{fjICCRCiQnK_eMJ2j8hOMg}{41h-4x5FSlWa1YYj2WCZfw}{172.22.3.172}{172.22.3.172:9300}{m}{rack_id=dom18, box_type=hot, xpack.installed=true, transform.node=false}
Hot threads at 2021-12-14T11:40:20.053Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-ingest18}{tgTgT-8TSXy4HxERqerbPQ}{D21hM1q7RzGppAv0un_dnQ}{172.22.3.178}{172.22.3.178:9300}{i}{rack_id=dom18, xpack.installed=true, box_type=hot, transform.node=false}
Hot threads at 2021-12-14T11:40:20.053Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (153.3micros out of 500ms) cpu usage by thread 'elasticsearch[esh1-ingest18][scheduler][T#1]'
3/10 snapshots sharing following 2 elements
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data11a}{wKRiS8wWRC64SOYqhExGow}{ZbupXWpdSku0aVK7kJ-hMw}{172.22.3.181}{172.22.3.181:9300}{cdfhstw}{rack_id=dom11, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.048Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (46.9micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data12b}{vnKZDhNmSoOKk8cbaio1OQ}{sdABetotTyC2D7IyRa8-FA}{172.22.3.186}{172.22.3.186:9300}{cdfhstw}{rack_id=dom12, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.052Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (25.9micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data13a}{URFM6bKeTICqn34N7hmlsA}{RDhHZQDGQ1-iNULr0wYqrw}{172.22.3.183}{172.22.3.183:9300}{cdfhstw}{rack_id=dom13, box_type=hot, xpack.installed=true, transform.node=true}
Hot threads at 2021-12-14T11:40:20.054Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (47.7micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data13c}{oTyaaQiuQk-7osC9yirMwQ}{ff-CmJO-TE6j8rNPa1076Q}{172.22.3.188}{172.22.3.188:9300}{cdfhstw}{rack_id=dom13, box_type=hot, xpack.installed=true, transform.node=true}
Hot threads at 2021-12-14T11:40:20.050Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.4% [cpu=100.4%, other=0.0%] (501.7ms out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13c][management][T#3]'
10/10 snapshots sharing following 65 elements
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl.getCpuLoad0(Native Method)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$SystemCpuTicks.lambda$defaultCpuLoadSupplier$0(OperatingSystemImpl.java:155)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$SystemCpuTicks$$Lambda$3014/0x0000000800feef80.getAsDouble(Unknown Source)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$ContainerCpuTicks.getContainerCpuLoad(OperatingSystemImpl.java:114)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl.getCpuLoad(OperatingSystemImpl.java:271)
jdk.management@17.0.1/com.sun.management.OperatingSystemMXBean.getSystemCpuLoad(OperatingSystemMXBean.java:153)
java.base@17.0.1/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base@17.0.1/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
java.base@17.0.1/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base@17.0.1/java.lang.reflect.Method.invoke(Method.java:568)
app//org.elasticsearch.monitor.Probes.getLoadAndScaleToPercent(Probes.java:18)
app//org.elasticsearch.monitor.os.OsProbe.getSystemCpuPercent(OsProbe.java:224)
app//org.elasticsearch.monitor.os.OsProbe.osStats(OsProbe.java:865)
app//org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:62)
app//org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:55)
app//org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:42)
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:391)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5749/0x0000000801914928.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:967)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:931)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:405)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5751/0x0000000801914d98.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:168)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:395)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:371)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:256)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5689/0x00000008018bc278.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:138)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5691/0x00000008018bc6d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:144)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:258)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5621/0x000000080178f2f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (42.6micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13c][management][T#1]'
10/10 snapshots sharing following 27 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:174)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:620)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:250)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5621/0x000000080178f2f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13c][management][T#4]'
10/10 snapshots sharing following 48 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:174)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:391)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5749/0x0000000801914928.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:967)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:931)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:405)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5751/0x0000000801914d98.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:168)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:395)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:371)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:256)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5689/0x00000008018bc278.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:138)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5691/0x00000008018bc6d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:144)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:258)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5621/0x000000080178f2f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13c][management][T#5]'
10/10 snapshots sharing following 49 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:391)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5749/0x0000000801914928.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:967)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:931)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:405)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5751/0x0000000801914d98.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:168)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:395)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:371)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:256)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5689/0x00000008018bc278.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:138)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5691/0x00000008018bc6d8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:144)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:258)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5621/0x000000080178f2f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13c][management][T#2]'
10/10 snapshots sharing following 28 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:620)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:250)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5621/0x000000080178f2f8.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1045)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data13d}{aYBnU37KQDK36-KjKflRXg}{6HnehnTxQRGza3DqrEvNgw}{172.22.3.163}{172.22.3.163:9300}{cdfhstw}{rack_id=dom13, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.050Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (43.9micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data11d}{CzooKBzzSZ2yNZ6r1IEFbw}{3bczmRTWS3SUvV5IuFAI4g}{172.22.3.161}{172.22.3.161:9300}{cdfhstw}{rack_id=dom11, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.049Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (46.3micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data11d][flush][T#5]'
2/10 snapshots sharing following 7 elements
app//org.elasticsearch.index.shard.IndexShard.flush(IndexShard.java:1420)
app//org.elasticsearch.indices.flush.SyncedFlushService$2.doRun(SyncedFlushService.java:155)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
unique snapshot
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
java.base@17.0.1/java.util.concurrent.LinkedTransferQueue$Node.block(LinkedTransferQueue.java:470)
java.base@17.0.1/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
java.base@17.0.1/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
java.base@17.0.1/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:669)
java.base@17.0.1/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:616)
java.base@17.0.1/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1286)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data12a}{aq2Vcn0xSOSOtxOvzMtdiQ}{el3dEksmTLS8RXMLr7J7EQ}{172.22.3.185}{172.22.3.185:9300}{cdfhstw}{rack_id=dom12, box_type=hot, xpack.installed=true, transform.node=true}
Hot threads at 2021-12-14T11:40:20.052Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (52.7micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-master1}{CIKwcfrBQCu7aqjQJ-u7Lg}{DDlPcXmkQFmhOY-PPrm6Fg}{172.22.3.170}{172.22.3.170:9300}{m}{rack_id=dom16, xpack.installed=true, box_type=hot, transform.node=false}
Hot threads at 2021-12-14T11:40:20.050Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-ingest16}{wQv5r7sBSYOu8X_KIIi1oQ}{Tc1HOu0-QaiEpn95XFMFxQ}{172.22.3.176}{172.22.3.176:9300}{i}{rack_id=dom16, xpack.installed=true, box_type=hot, transform.node=false}
Hot threads at 2021-12-14T11:40:20.051Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-ingest17}{LmpUqIihR3-3wb9fAU_CJA}{Dgn_MmeJTpSDhW1NCsqu0Q}{172.22.3.177}{172.22.3.177:9300}{i}{rack_id=dom17, xpack.installed=true, box_type=hot, transform.node=false}
Hot threads at 2021-12-14T11:40:20.047Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-master2}{m8l9CsbURKist_deyFL9nw}{Ga6LAYp6Sei0oyisTiFPeQ}{172.22.3.171}{172.22.3.171:9300}{m}{rack_id=dom17, box_type=hot, xpack.installed=true, transform.node=false}
Hot threads at 2021-12-14T11:40:20.048Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-ingest19}{HfVBOU3DRLaSk2WjkM4k7Q}{TgMcMeN5Rdud-K_WgCWsmA}{172.22.3.168}{172.22.3.168:9300}{i}{rack_id=dom19, xpack.installed=true, box_type=hot, transform.node=false}
Hot threads at 2021-12-14T11:40:20.051Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
::: {esh1-data13b}{iMhFSXQ2SVSKuzpAWPLomw}{dq8CC5rYSlygcB0Gvst3ng}{172.22.3.184}{172.22.3.184:9300}{cdfhstw}{rack_id=dom13, xpack.installed=true, box_type=hot, transform.node=true}
Hot threads at 2021-12-14T11:40:20.051Z, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
100.3% [cpu=100.3%, other=0.0%] (501.6ms out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13b][management][T#4]'
10/10 snapshots sharing following 62 elements
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl.getProcessCpuLoad0(Native Method)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$ProcessCpuTicks.lambda$defaultCpuLoadSupplier$0(OperatingSystemImpl.java:132)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$ProcessCpuTicks$$Lambda$3018/0x0000000800fef1a8.getAsDouble(Unknown Source)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$ContainerCpuTicks.getContainerCpuLoad(OperatingSystemImpl.java:114)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl.getProcessCpuLoad(OperatingSystemImpl.java:278)
java.base@17.0.1/jdk.internal.reflect.GeneratedMethodAccessor15.invoke(Unknown Source)
java.base@17.0.1/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base@17.0.1/java.lang.reflect.Method.invoke(Method.java:568)
app//org.elasticsearch.monitor.Probes.getLoadAndScaleToPercent(Probes.java:18)
app//org.elasticsearch.monitor.process.ProcessProbe.getProcessCpuPercent(ProcessProbe.java:80)
app//org.elasticsearch.monitor.process.ProcessProbe.processStats(ProcessProbe.java:122)
app//org.elasticsearch.monitor.process.ProcessService$ProcessStatsCache.refresh(ProcessService.java:59)
app//org.elasticsearch.monitor.process.ProcessService$ProcessStatsCache.refresh(ProcessService.java:52)
app//org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:42)
app//org.elasticsearch.monitor.process.ProcessService.stats(ProcessService.java:49)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:151)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:174)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:391)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5796/0x000000080197d410.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:967)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:931)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:405)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5798/0x000000080197d880.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:168)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:395)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:371)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:256)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5769/0x00000008018b4fc0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:138)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5771/0x00000008018b5420.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:144)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:258)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5629/0x00000008018b4208.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (44.3micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
10/10 snapshots sharing following 2 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:196)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/7.16.1]'
10/10 snapshots sharing following 7 elements
java.base@17.0.1/jdk.internal.misc.Unsafe.park(Native Method)
java.base@17.0.1/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
java.base@17.0.1/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
java.base@17.0.1/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:85)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'Connection evictor'
10/10 snapshots sharing following 3 elements
java.base@17.0.1/java.lang.Thread.sleep(Native Method)
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13b][management][T#1]'
10/10 snapshots sharing following 28 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:620)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:250)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5629/0x00000008018b4208.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1045)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13b][management][T#3]'
10/10 snapshots sharing following 27 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:174)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:620)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:250)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5629/0x00000008018b4208.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13b][management][T#5]'
10/10 snapshots sharing following 63 elements
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl.getCpuLoad0(Native Method)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$SystemCpuTicks.lambda$defaultCpuLoadSupplier$0(OperatingSystemImpl.java:155)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$SystemCpuTicks$$Lambda$3014/0x0000000800feef80.getAsDouble(Unknown Source)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl$ContainerCpuTicks.getContainerCpuLoad(OperatingSystemImpl.java:114)
jdk.management@17.0.1/com.sun.management.internal.OperatingSystemImpl.getCpuLoad(OperatingSystemImpl.java:271)
jdk.management@17.0.1/com.sun.management.OperatingSystemMXBean.getSystemCpuLoad(OperatingSystemMXBean.java:153)
java.base@17.0.1/jdk.internal.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
java.base@17.0.1/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.base@17.0.1/java.lang.reflect.Method.invoke(Method.java:568)
app//org.elasticsearch.monitor.Probes.getLoadAndScaleToPercent(Probes.java:18)
app//org.elasticsearch.monitor.os.OsProbe.getSystemCpuPercent(OsProbe.java:224)
app//org.elasticsearch.monitor.os.OsProbe.osStats(OsProbe.java:865)
app//org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:62)
app//org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:55)
app//org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:42)
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:174)
app//org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:54)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:391)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5796/0x000000080197d410.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:967)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:931)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:405)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5798/0x000000080197d880.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:168)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:395)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:371)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:256)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5769/0x00000008018b4fc0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:138)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5771/0x00000008018b5420.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:144)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:258)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5629/0x00000008018b4208.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
0.0% [cpu=0.0%, other=0.0%] (0s out of 500ms) cpu usage by thread 'elasticsearch[esh1-data13b][management][T#2]'
10/10 snapshots sharing following 49 elements
app//org.elasticsearch.monitor.os.OsService.stats(OsService.java:52)
app//org.elasticsearch.node.NodeService.stats(NodeService.java:150)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:87)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:80)
app//org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:32)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:305)
app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:299)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:341)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:404)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:394)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:391)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5796/0x000000080197d410.accept(Unknown Source)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:967)
org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:931)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:405)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5798/0x000000080197d880.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:168)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:395)
org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:371)
org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:256)
org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$5769/0x00000008018b4fc0.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31)
org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:138)
org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$5771/0x00000008018b5420.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.roles(CompositeRolesStore.java:173)
org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:279)
org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:144)
org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:127)
org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:258)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:136)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$5629/0x00000008018b4208.accept(Unknown Source)
app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136)
app//org.elasticsearch.action.ActionListener$MappedActionListener.onResponse(ActionListener.java:101)
org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticateAsync(AuthenticatorChain.java:102)
org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:199)
org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:128)
org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:415)
app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:67)
app//org.elasticsearch.transport.InboundHandler$1.doRun(InboundHandler.java:260)
app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:777)
app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
java.base@17.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
java.base@17.0.1/java.lang.Thread.run(Thread.java:833)
esh1-data13c esh1.log
[2021-12-14T11:30:18,219][ERROR][o.e.x.m.c.n.NodeStatsCollector] [esh1-data13c] collector [node_stats] timed out when collecting data: node [oTyaaQiuQk-7osC9yirMwQ] did not respond within [30s]
[2021-12-14T11:32:18,218][ERROR][o.e.x.m.c.n.NodeStatsCollector] [esh1-data13c] collector [node_stats] timed out when collecting data: node [oTyaaQiuQk-7osC9yirMwQ] did not respond within [30s]
[2021-12-14T11:34:18,219][ERROR][o.e.x.m.c.n.NodeStatsCollector] [esh1-data13c] collector [node_stats] timed out when collecting data: node [oTyaaQiuQk-7osC9yirMwQ] did not respond within [30s]
[2021-12-14T11:36:18,219][ERROR][o.e.x.m.c.n.NodeStatsCollector] [esh1-data13c] collector [node_stats] timed out when collecting data: node [oTyaaQiuQk-7osC9yirMwQ] did not respond within [30s]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment