Created
September 28, 2012 18:34
-
-
Save alpha01/3801427 to your computer and use it in GitHub Desktop.
Node logger-rdc-2 that is part of the cluster. Output of when a different node called logger-rdc-3, was having extremely high cpu load
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "ok" : true, | |
| "_shards" : { | |
| "total" : 10, | |
| "successful" : 9, | |
| "failed" : 1, | |
| "failures" : [ { | |
| "index" : "graylog2", | |
| "shard" : 2, | |
| "reason" : "BroadcastShardOperationFailedException[[graylog2][2] ]; nested: RemoteTransportException[[logger-rdc-3][inet[/192.168.4.13:9300]][indices/status/s]]; nested: IndexShardMissingException[[graylog2][2] missing]; " | |
| } ] | |
| }, | |
| "indices" : { | |
| "graylog2" : { | |
| "index" : { | |
| "primary_size" : "39gb", | |
| "primary_size_in_bytes" : 41936864933, | |
| "size" : "67.5gb", | |
| "size_in_bytes" : 72499514175 | |
| }, | |
| "translog" : { | |
| "operations" : 6549 | |
| }, | |
| "docs" : { | |
| "num_docs" : 20862397, | |
| "max_doc" : 24118741, | |
| "deleted_docs" : 3256344 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 50353, | |
| "total_time" : "1.7h", | |
| "total_time_in_millis" : 6126011, | |
| "total_docs" : 48437383, | |
| "total_size" : "81.2gb", | |
| "total_size_in_bytes" : 87187981674 | |
| }, | |
| "refresh" : { | |
| "total" : 524982, | |
| "total_time" : "1.3h", | |
| "total_time_in_millis" : 4682395 | |
| }, | |
| "flush" : { | |
| "total" : 562, | |
| "total_time" : "1.4m", | |
| "total_time_in_millis" : 84972 | |
| }, | |
| "shards" : { | |
| "0" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "qG8xORUDQlCzg6AfEmBe9w", | |
| "relocating_node" : null, | |
| "shard" : 0, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.5gb", | |
| "size_in_bytes" : 7003864199 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333002, | |
| "operations" : 260 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4163531, | |
| "max_doc" : 4227435, | |
| "deleted_docs" : 63904 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 238, | |
| "total_time" : "14.9s", | |
| "total_time_in_millis" : 14999, | |
| "total_docs" : 73099, | |
| "total_size" : "136.4mb", | |
| "total_size_in_bytes" : 143056251 | |
| }, | |
| "refresh" : { | |
| "total" : 2133, | |
| "total_time" : "17.4s", | |
| "total_time_in_millis" : 17425 | |
| }, | |
| "flush" : { | |
| "total" : 2, | |
| "total_time" : "230ms", | |
| "total_time_in_millis" : 230 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "4-KejzGOTTmj0KblsBVQmw", | |
| "relocating_node" : null, | |
| "shard" : 0, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.5gb", | |
| "size_in_bytes" : 6992337792 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333002, | |
| "operations" : 32 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4158296, | |
| "max_doc" : 4222200, | |
| "deleted_docs" : 63904 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 8070, | |
| "total_time" : "12.1m", | |
| "total_time_in_millis" : 728853, | |
| "total_docs" : 6245729, | |
| "total_size" : "10.6gb", | |
| "total_size_in_bytes" : 11463719443 | |
| }, | |
| "refresh" : { | |
| "total" : 72598, | |
| "total_time" : "7.1m", | |
| "total_time_in_millis" : 429570 | |
| }, | |
| "flush" : { | |
| "total" : 79, | |
| "total_time" : "10.5s", | |
| "total_time_in_millis" : 10558 | |
| } | |
| } ], | |
| "1" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "4-KejzGOTTmj0KblsBVQmw", | |
| "relocating_node" : null, | |
| "shard" : 1, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "9gb", | |
| "size_in_bytes" : 9687836906 | |
| }, | |
| "translog" : { | |
| "id" : 1335913332936, | |
| "operations" : 0 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4163586, | |
| "max_doc" : 5747090, | |
| "deleted_docs" : 1583504 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 0, | |
| "total_time" : "0s", | |
| "total_time_in_millis" : 0, | |
| "total_docs" : 0, | |
| "total_size" : "0b", | |
| "total_size_in_bytes" : 0 | |
| }, | |
| "refresh" : { | |
| "total" : 72075, | |
| "total_time" : "7.7m", | |
| "total_time_in_millis" : 464765 | |
| }, | |
| "flush" : { | |
| "total" : 79, | |
| "total_time" : "11.3s", | |
| "total_time_in_millis" : 11363 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 1, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "9gb", | |
| "size_in_bytes" : 9705960783 | |
| }, | |
| "translog" : { | |
| "id" : 1335913332938, | |
| "operations" : 876 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4167961, | |
| "max_doc" : 5752809, | |
| "deleted_docs" : 1584848 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 13567, | |
| "total_time" : "25.1m", | |
| "total_time_in_millis" : 1510473, | |
| "total_docs" : 10995215, | |
| "total_size" : "18.6gb", | |
| "total_size_in_bytes" : 20055423836 | |
| }, | |
| "refresh" : { | |
| "total" : 122027, | |
| "total_time" : "20m", | |
| "total_time_in_millis" : 1201908 | |
| }, | |
| "flush" : { | |
| "total" : 127, | |
| "total_time" : "17.9s", | |
| "total_time_in_millis" : 17969 | |
| } | |
| } ], | |
| "2" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 2, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "10.5gb", | |
| "size_in_bytes" : 11364999542 | |
| }, | |
| "translog" : { | |
| "id" : 1335913332985, | |
| "operations" : 897 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4172475, | |
| "max_doc" : 5768741, | |
| "deleted_docs" : 1596266 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 563, | |
| "total_time" : "56.8s", | |
| "total_time_in_millis" : 56898, | |
| "total_docs" : 395844, | |
| "total_size" : "696.6mb", | |
| "total_size_in_bytes" : 730451639 | |
| }, | |
| "refresh" : { | |
| "total" : 5066, | |
| "total_time" : "40s", | |
| "total_time_in_millis" : 40013 | |
| }, | |
| "flush" : { | |
| "total" : 4, | |
| "total_time" : "516ms", | |
| "total_time_in_millis" : 516 | |
| } | |
| } ], | |
| "3" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "qG8xORUDQlCzg6AfEmBe9w", | |
| "relocating_node" : null, | |
| "shard" : 3, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6949807766 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333332, | |
| "operations" : 1693 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4179806, | |
| "max_doc" : 4184731, | |
| "deleted_docs" : 4925 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 13544, | |
| "total_time" : "31.3m", | |
| "total_time_in_millis" : 1879647, | |
| "total_docs" : 15151977, | |
| "total_size" : "25.1gb", | |
| "total_size_in_bytes" : 26956707453 | |
| }, | |
| "refresh" : { | |
| "total" : 121816, | |
| "total_time" : "21.6m", | |
| "total_time_in_millis" : 1301611 | |
| }, | |
| "flush" : { | |
| "total" : 130, | |
| "total_time" : "22.2s", | |
| "total_time_in_millis" : 22234 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 3, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6947961891 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333332, | |
| "operations" : 1160 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4179805, | |
| "max_doc" : 4184730, | |
| "deleted_docs" : 4925 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 283, | |
| "total_time" : "39.5s", | |
| "total_time_in_millis" : 39542, | |
| "total_docs" : 282134, | |
| "total_size" : "498.4mb", | |
| "total_size_in_bytes" : 522701735 | |
| }, | |
| "refresh" : { | |
| "total" : 2539, | |
| "total_time" : "19.2s", | |
| "total_time_in_millis" : 19275 | |
| }, | |
| "flush" : { | |
| "total" : 2, | |
| "total_time" : "460ms", | |
| "total_time_in_millis" : 460 | |
| } | |
| } ], | |
| "4" : [ { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : false, | |
| "node" : "qG8xORUDQlCzg6AfEmBe9w", | |
| "relocating_node" : null, | |
| "shard" : 4, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6922986246 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333346, | |
| "operations" : 1010 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4183859, | |
| "max_doc" : 4190260, | |
| "deleted_docs" : 6401 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 906, | |
| "total_time" : "1.9m", | |
| "total_time_in_millis" : 117625, | |
| "total_docs" : 667513, | |
| "total_size" : "1.1gb", | |
| "total_size_in_bytes" : 1246455463 | |
| }, | |
| "refresh" : { | |
| "total" : 8145, | |
| "total_time" : "1.4m", | |
| "total_time_in_millis" : 89334 | |
| }, | |
| "flush" : { | |
| "total" : 12, | |
| "total_time" : "1.6s", | |
| "total_time_in_millis" : 1648 | |
| } | |
| }, { | |
| "routing" : { | |
| "state" : "STARTED", | |
| "primary" : true, | |
| "node" : "_GNgOXhGRJ-I0c_aZTP1Fg", | |
| "relocating_node" : null, | |
| "shard" : 4, | |
| "index" : "graylog2" | |
| }, | |
| "state" : "STARTED", | |
| "index" : { | |
| "size" : "6.4gb", | |
| "size_in_bytes" : 6923759050 | |
| }, | |
| "translog" : { | |
| "id" : 1335913333347, | |
| "operations" : 621 | |
| }, | |
| "docs" : { | |
| "num_docs" : 4183859, | |
| "max_doc" : 4190260, | |
| "deleted_docs" : 6401 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 13182, | |
| "total_time" : "29.6m", | |
| "total_time_in_millis" : 1777974, | |
| "total_docs" : 14625872, | |
| "total_size" : "24.2gb", | |
| "total_size_in_bytes" : 26069465854 | |
| }, | |
| "refresh" : { | |
| "total" : 118583, | |
| "total_time" : "18.6m", | |
| "total_time_in_millis" : 1118494 | |
| }, | |
| "flush" : { | |
| "total" : 127, | |
| "total_time" : "19.9s", | |
| "total_time_in_millis" : 19994 | |
| } | |
| } ] | |
| } | |
| } | |
| } | |
| }{ | |
| "cluster_name" : "logger-rdc-elasticsearch", | |
| "nodes" : { | |
| "0sVfY5EaTZGdejJSAPAmqg" : { | |
| "name" : "logger-rdc-2", | |
| "transport_address" : "inet[/192.168.4.12:9300]", | |
| "hostname" : "logger-rdc-2", | |
| "indices" : { | |
| "store" : { | |
| "size" : "0b", | |
| "size_in_bytes" : 0 | |
| }, | |
| "docs" : { | |
| "count" : 0, | |
| "deleted" : 0 | |
| }, | |
| "indexing" : { | |
| "index_total" : 0, | |
| "index_time" : "0s", | |
| "index_time_in_millis" : 0, | |
| "index_current" : 0, | |
| "delete_total" : 0, | |
| "delete_time" : "0s", | |
| "delete_time_in_millis" : 0, | |
| "delete_current" : 0 | |
| }, | |
| "get" : { | |
| "total" : 0, | |
| "time" : "0s", | |
| "time_in_millis" : 0, | |
| "exists_total" : 0, | |
| "exists_time" : "0s", | |
| "exists_time_in_millis" : 0, | |
| "missing_total" : 0, | |
| "missing_time" : "0s", | |
| "missing_time_in_millis" : 0, | |
| "current" : 0 | |
| }, | |
| "search" : { | |
| "query_total" : 0, | |
| "query_time" : "0s", | |
| "query_time_in_millis" : 0, | |
| "query_current" : 0, | |
| "fetch_total" : 0, | |
| "fetch_time" : "0s", | |
| "fetch_time_in_millis" : 0, | |
| "fetch_current" : 0 | |
| }, | |
| "cache" : { | |
| "field_evictions" : 0, | |
| "field_size" : "0b", | |
| "field_size_in_bytes" : 0, | |
| "filter_count" : 0, | |
| "filter_evictions" : 0, | |
| "filter_size" : "0b", | |
| "filter_size_in_bytes" : 0 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size" : "0b", | |
| "current_size_in_bytes" : 0, | |
| "total" : 0, | |
| "total_time" : "0s", | |
| "total_time_in_millis" : 0, | |
| "total_docs" : 0, | |
| "total_size" : "0b", | |
| "total_size_in_bytes" : 0 | |
| }, | |
| "refresh" : { | |
| "total" : 0, | |
| "total_time" : "0s", | |
| "total_time_in_millis" : 0 | |
| }, | |
| "flush" : { | |
| "total" : 0, | |
| "total_time" : "0s", | |
| "total_time_in_millis" : 0 | |
| } | |
| } | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment