Created
September 28, 2012 18:37
-
-
Save alpha01/3801445 to your computer and use it in GitHub Desktop.
Node logger-rdc-3, output of when Elasticsearch was experiencing an extrememly high cpu load.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "ok" : true, | |
| "_shards" : { | |
| "total" : 10, | |
| "successful" : 9, | |
| "failed" : 1, | |
| "failures" : [ { | |
| "index" : "graylog2", | |
| "shard" : 2, | |
| "reason" : "BroadcastShardOperationFailedException[[graylog2][2] ]; nested: IndexShardMissingException[[graylog2][2] missing]; " | |
| } ] | |
| }, | |
| "indices" : { | |
| "graylog2" : { | |
| "index" : { | |
| "primary_size" : "39gb", | |
| "primary_size_in_bytes" : 41936698607, | |
| "size" : "67.5gb", | |
| "size_in_bytes" : 72499258477 | |
| }, | |
| "translog" : { | |
| "operations" : 6401 | |
| }, | |
| "docs" : { | |
| "num_docs" : 20862313, | |
| "max_doc" : 24118657, | |
| "deleted_docs" : 3256344 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 50342, | |
| "total_time" : "1.7h", | |
| "total_time_in_millis" : 6125369, | |
| "total_docs" : 48433860, | |
| "total_size" : "81.1gb", | |
| "total_size_in_bytes" : 87181102220 | |
| }, | |
| "refresh" : { | |
| "total" : 524889, | |
| "total_time" : "1.3h", | |
| "total_time_in_millis" : 4681811 | |
| }, | |
| "flush" : { | |
| "total" : 562, | |
| "total_time" : "1.4m", | |
| "total_time_in_millis" : 84972 | |
| }, | |
| "shards" : { | |
| "0" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "qG8xORUDQlCzg6AfEmBe9w", | |
| "relocating_node" : null, | |
| "shard" : 0, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.5gb", | |
| "size_in_bytes" : 7003840278 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333002, | |
| "operations" : 239 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4163511, | |
| "max_doc" : 4227415, | |
| "deleted_docs" : 63904 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 236, | |
| "total_time" : "14.8s", | |
| "total_time_in_millis" : 14857, | |
| "total_docs" : 72051, | |
| "total_size" : "134.5mb", | |
| "total_size_in_bytes" : 141067663 | |
| }, | |
| "refresh" : { | |
| "total" : 2120, | |
| "total_time" : "17.3s", | |
| "total_time_in_millis" : 17352 | |
| }, | |
| "flush" : { | |
| "total" : 2, | |
| "total_time" : "230ms", | |
| "total_time_in_millis" : 230 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "4-KejzGOTTmj0KblsBVQmw", | |
| "relocating_node" : null, | |
| "shard" : 0, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.5gb", | |
| "size_in_bytes" : 6992335019 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333002, | |
| "operations" : 31 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4158295, | |
| "max_doc" : 4222199, | |
| "deleted_docs" : 63904 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 8070, | |
| "total_time" : "12.1m", | |
| "total_time_in_millis" : 728853, | |
| "total_docs" : 6245729, | |
| "total_size" : "10.6gb", | |
| "total_size_in_bytes" : 11463719443 | |
| }, | |
| "refresh" : { | |
| "total" : 72597, | |
| "total_time" : "7.1m", | |
| "total_time_in_millis" : 429563 | |
| }, | |
| "flush" : { | |
| "total" : 79, | |
| "total_time" : "10.5s", | |
| "total_time_in_millis" : 10558 | |
| } | |
| } ], | |
| "1" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 1, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "9gb", | |
| "size_in_bytes" : 9705909490 | |
| }, | |
| "translog" : { | |
| "id" : 1335913332938, | |
| "operations" : 854 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4167939, | |
| "max_doc" : 5752787, | |
| "deleted_docs" : 1584848 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 13566, | |
| "total_time" : "25.1m", | |
| "total_time_in_millis" : 1510397, | |
| "total_docs" : 10994806, | |
| "total_size" : "18.6gb", | |
| "total_size_in_bytes" : 20054642744 | |
| }, | |
| "refresh" : { | |
| "total" : 122013, | |
| "total_time" : "20m", | |
| "total_time_in_millis" : 1201820 | |
| }, | |
| "flush" : { | |
| "total" : 127, | |
| "total_time" : "17.9s", | |
| "total_time_in_millis" : 17969 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "4-KejzGOTTmj0KblsBVQmw", | |
| "relocating_node" : null, | |
| "shard" : 1, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "9gb", | |
| "size_in_bytes" : 9687836906 | |
| }, | |
| "translog" : { | |
| "id" : 1335913332936, | |
| "operations" : 0 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4163586, | |
| "max_doc" : 5747090, | |
| "deleted_docs" : 1583504 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 0, | |
| "total_time" : "0s", | |
| "total_time_in_millis" : 0, | |
| "total_docs" : 0, | |
| "total_size" : "0b", | |
| "total_size_in_bytes" : 0 | |
| }, | |
| "refresh" : { | |
| "total" : 72075, | |
| "total_time" : "7.7m", | |
| "total_time_in_millis" : 464765 | |
| }, | |
| "flush" : { | |
| "total" : 79, | |
| "total_time" : "11.3s", | |
| "total_time_in_millis" : 11363 | |
| } | |
| } ], | |
| "2" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 2, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "10.5gb", | |
| "size_in_bytes" : 11364976337 | |
| }, | |
| "translog" : { | |
| "id" : 1335913332985, | |
| "operations" : 883 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4172461, | |
| "max_doc" : 5768727, | |
| "deleted_docs" : 1596266 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 562, | |
| "total_time" : "56.8s", | |
| "total_time_in_millis" : 56824, | |
| "total_docs" : 395368, | |
| "total_size" : "695.7mb", | |
| "total_size_in_bytes" : 729536122 | |
| }, | |
| "refresh" : { | |
| "total" : 5057, | |
| "total_time" : "39.9s", | |
| "total_time_in_millis" : 39951 | |
| }, | |
| "flush" : { | |
| "total" : 4, | |
| "total_time" : "516ms", | |
| "total_time_in_millis" : 516 | |
| } | |
| } ], | |
| "3" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 3, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6947927949 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333332, | |
| "operations" : 1136 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4179780, | |
| "max_doc" : 4184705, | |
| "deleted_docs" : 4925 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 281, | |
| "total_time" : "39.4s", | |
| "total_time_in_millis" : 39464, | |
| "total_docs" : 281958, | |
| "total_size" : "498.1mb", | |
| "total_size_in_bytes" : 522325012 | |
| }, | |
| "refresh" : { | |
| "total" : 2526, | |
| "total_time" : "19.1s", | |
| "total_time_in_millis" : 19185 | |
| }, | |
| "flush" : { | |
| "total" : 2, | |
| "total_time" : "460ms", | |
| "total_time_in_millis" : 460 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "qG8xORUDQlCzg6AfEmBe9w", | |
| "relocating_node" : null, | |
| "shard" : 3, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6949773667 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333332, | |
| "operations" : 1669 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4179780, | |
| "max_doc" : 4184705, | |
| "deleted_docs" : 4925 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 13542, | |
| "total_time" : "31.3m", | |
| "total_time_in_millis" : 1879555, | |
| "total_docs" : 15151756, | |
| "total_size" : "25.1gb", | |
| "total_size_in_bytes" : 26956215116 | |
| }, | |
| "refresh" : { | |
| "total" : 121803, | |
| "total_time" : "21.6m", | |
| "total_time_in_millis" : 1301540 | |
| }, | |
| "flush" : { | |
| "total" : 130, | |
| "total_time" : "22.2s", | |
| "total_time_in_millis" : 22234 | |
| } | |
| } ], | |
| "4" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 4, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6923704094 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333347, | |
| "operations" : 600 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4183838, | |
| "max_doc" : 4190239, | |
| "deleted_docs" : 6401 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 13181, | |
| "total_time" : "29.6m", | |
| "total_time_in_millis" : 1777902, | |
| "total_docs" : 14625472, | |
| "total_size" : "24.2gb", | |
| "total_size_in_bytes" : 26068689802 | |
| }, | |
| "refresh" : { | |
| "total" : 118568, | |
| "total_time" : "18.6m", | |
| "total_time_in_millis" : 1118381 | |
| }, | |
| "flush" : { | |
| "total" : 127, | |
| "total_time" : "19.9s", | |
| "total_time_in_millis" : 19994 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "qG8xORUDQlCzg6AfEmBe9w", | |
| "relocating_node" : null, | |
| "shard" : 4, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6922954737 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333346, | |
| "operations" : 989 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4183838, | |
| "max_doc" : 4190239, | |
| "deleted_docs" : 6401 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 904, | |
| "total_time" : "1.9m", | |
| "total_time_in_millis" : 117517, | |
| "total_docs" : 666720, | |
| "total_size" : "1.1gb", | |
| "total_size_in_bytes" : 1244906318 | |
| }, | |
| "refresh" : { | |
| "total" : 8130, | |
| "total_time" : "1.4m", | |
| "total_time_in_millis" : 89254 | |
| }, | |
| "flush" : { | |
| "total" : 12, | |
| "total_time" : "1.6s", | |
| "total_time_in_millis" : 1648 | |
| } | |
| } ] | |
| } | |
| } | |
| } | |
| }{ | |
| "cluster_name" : "logger-rdc-elasticsearch", | |
| "nodes" : { | |
| "4-KejzGOTTmj0KblsBVQmw" : { | |
| "name" : "logger-rdc-3", | |
| "transport_address" : "inet[/192.168.4.13:9300]", | |
| "hostname" : "logger-rdc-3", | |
| "indices" : { | |
| "store" : { | |
| "size" : "15.5gb", | |
| "size_in_bytes" : 16680171925 | |
| }, | |
| "docs" : { | |
| "count" : 8321881, | |
| "deleted" : 1647408 | |
| }, | |
| "indexing" : { | |
| "index_total" : 237587, | |
| "index_time" : "5.1m", | |
| "index_time_in_millis" : 310186, | |
| "index_current" : 0, | |
| "delete_total" : 0, | |
| "delete_time" : "0s", | |
| "delete_time_in_millis" : 0, | |
| "delete_current" : 0 | |
| }, | |
| "get" : { | |
| "total" : 3, | |
| "time" : "3ms", | |
| "time_in_millis" : 3, | |
| "exists_total" : 3, | |
| "exists_time" : "3ms", | |
| "exists_time_in_millis" : 3, | |
| "missing_total" : 0, | |
| "missing_time" : "0s", | |
| "missing_time_in_millis" : 0, | |
| "current" : 0 | |
| }, | |
| "search" : { | |
| "query_total" : 18, | |
| "query_time" : "10.3s", | |
| "query_time_in_millis" : 10361, | |
| "query_current" : 0, | |
| "fetch_total" : 18, | |
| "fetch_time" : "59ms", | |
| "fetch_time_in_millis" : 59, | |
| "fetch_current" : 0 | |
| }, | |
| "cache" : { | |
| "field_evictions" : 0, | |
| "field_size" : "34.8mb", | |
| "field_size_in_bytes" : 36551964, | |
| "filter_count" : 1, | |
| "filter_evictions" : 0, | |
| "filter_size" : "515.2kb", | |
| "filter_size_in_bytes" : 527624 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 8070, | |
| "total_time" : "12.1m", | |
| "total_time_in_millis" : 728853, | |
| "total_docs" : 6245729, | |
| "total_size" : "10.6gb", | |
| "total_size_in_bytes" : 11463719443 | |
| }, | |
| "refresh" : { | |
| "total" : 144672, | |
| "total_time" : "14.9m", | |
| "total_time_in_millis" : 894328 | |
| }, | |
| "flush" : { | |
| "total" : 158, | |
| "total_time" : "21.9s", | |
| "total_time_in_millis" : 21921 | |
| } | |
| } | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment