Skip to content

Instantly share code, notes, and snippets.

@alpha01
Created September 28, 2012 18:37
Show Gist options
  • Save alpha01/3801445 to your computer and use it in GitHub Desktop.
Save alpha01/3801445 to your computer and use it in GitHub Desktop.
Node logger-rdc-3, output of when Elasticsearch was experiencing an extrememly high cpu load.
{
"ok" : true,
"_shards" : {
"total" : 10,
"successful" : 9,
"failed" : 1,
"failures" : [ {
"index" : "graylog2",
"shard" : 2,
"reason" : "BroadcastShardOperationFailedException[[graylog2][2] ]; nested: IndexShardMissingException[[graylog2][2] missing]; "
} ]
},
"indices" : {
"graylog2" : {
"index" : {
"primary_size" : "39gb",
"primary_size_in_bytes" : 41936698607,
"size" : "67.5gb",
"size_in_bytes" : 72499258477
},
"translog" : {
"operations" : 6401
},
"docs" : {
"num_docs" : 20862313,
"max_doc" : 24118657,
"deleted_docs" : 3256344
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 50342,
"total_time" : "1.7h",
"total_time_in_millis" : 6125369,
"total_docs" : 48433860,
"total_size" : "81.1gb",
"total_size_in_bytes" : 87181102220
},
"refresh" : {
"total" : 524889,
"total_time" : "1.3h",
"total_time_in_millis" : 4681811
},
"flush" : {
"total" : 562,
"total_time" : "1.4m",
"total_time_in_millis" : 84972
},
"shards" : {
"0" : [ {
"routing" : {
"state" : "STARTED",
"primary" : false,
"node" : "qG8xORUDQlCzg6AfEmBe9w",
"relocating_node" : null,
"shard" : 0,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "6.5gb",
"size_in_bytes" : 7003840278
},
"translog" : {
"id" : 1335913333002,
"operations" : 239
},
"docs" : {
"num_docs" : 4163511,
"max_doc" : 4227415,
"deleted_docs" : 63904
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 236,
"total_time" : "14.8s",
"total_time_in_millis" : 14857,
"total_docs" : 72051,
"total_size" : "134.5mb",
"total_size_in_bytes" : 141067663
},
"refresh" : {
"total" : 2120,
"total_time" : "17.3s",
"total_time_in_millis" : 17352
},
"flush" : {
"total" : 2,
"total_time" : "230ms",
"total_time_in_millis" : 230
}
}, {
"routing" : {
"state" : "STARTED",
"primary" : true,
"node" : "4-KejzGOTTmj0KblsBVQmw",
"relocating_node" : null,
"shard" : 0,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "6.5gb",
"size_in_bytes" : 6992335019
},
"translog" : {
"id" : 1335913333002,
"operations" : 31
},
"docs" : {
"num_docs" : 4158295,
"max_doc" : 4222199,
"deleted_docs" : 63904
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 8070,
"total_time" : "12.1m",
"total_time_in_millis" : 728853,
"total_docs" : 6245729,
"total_size" : "10.6gb",
"total_size_in_bytes" : 11463719443
},
"refresh" : {
"total" : 72597,
"total_time" : "7.1m",
"total_time_in_millis" : 429563
},
"flush" : {
"total" : 79,
"total_time" : "10.5s",
"total_time_in_millis" : 10558
}
} ],
"1" : [ {
"routing" : {
"state" : "STARTED",
"primary" : true,
"node" : "_GNgOXhGRJ-I0c_aZTP1Fg",
"relocating_node" : null,
"shard" : 1,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "9gb",
"size_in_bytes" : 9705909490
},
"translog" : {
"id" : 1335913332938,
"operations" : 854
},
"docs" : {
"num_docs" : 4167939,
"max_doc" : 5752787,
"deleted_docs" : 1584848
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 13566,
"total_time" : "25.1m",
"total_time_in_millis" : 1510397,
"total_docs" : 10994806,
"total_size" : "18.6gb",
"total_size_in_bytes" : 20054642744
},
"refresh" : {
"total" : 122013,
"total_time" : "20m",
"total_time_in_millis" : 1201820
},
"flush" : {
"total" : 127,
"total_time" : "17.9s",
"total_time_in_millis" : 17969
}
}, {
"routing" : {
"state" : "STARTED",
"primary" : false,
"node" : "4-KejzGOTTmj0KblsBVQmw",
"relocating_node" : null,
"shard" : 1,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "9gb",
"size_in_bytes" : 9687836906
},
"translog" : {
"id" : 1335913332936,
"operations" : 0
},
"docs" : {
"num_docs" : 4163586,
"max_doc" : 5747090,
"deleted_docs" : 1583504
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 0,
"total_time" : "0s",
"total_time_in_millis" : 0,
"total_docs" : 0,
"total_size" : "0b",
"total_size_in_bytes" : 0
},
"refresh" : {
"total" : 72075,
"total_time" : "7.7m",
"total_time_in_millis" : 464765
},
"flush" : {
"total" : 79,
"total_time" : "11.3s",
"total_time_in_millis" : 11363
}
} ],
"2" : [ {
"routing" : {
"state" : "STARTED",
"primary" : true,
"node" : "_GNgOXhGRJ-I0c_aZTP1Fg",
"relocating_node" : null,
"shard" : 2,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "10.5gb",
"size_in_bytes" : 11364976337
},
"translog" : {
"id" : 1335913332985,
"operations" : 883
},
"docs" : {
"num_docs" : 4172461,
"max_doc" : 5768727,
"deleted_docs" : 1596266
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 562,
"total_time" : "56.8s",
"total_time_in_millis" : 56824,
"total_docs" : 395368,
"total_size" : "695.7mb",
"total_size_in_bytes" : 729536122
},
"refresh" : {
"total" : 5057,
"total_time" : "39.9s",
"total_time_in_millis" : 39951
},
"flush" : {
"total" : 4,
"total_time" : "516ms",
"total_time_in_millis" : 516
}
} ],
"3" : [ {
"routing" : {
"state" : "STARTED",
"primary" : false,
"node" : "_GNgOXhGRJ-I0c_aZTP1Fg",
"relocating_node" : null,
"shard" : 3,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "6.4gb",
"size_in_bytes" : 6947927949
},
"translog" : {
"id" : 1335913333332,
"operations" : 1136
},
"docs" : {
"num_docs" : 4179780,
"max_doc" : 4184705,
"deleted_docs" : 4925
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 281,
"total_time" : "39.4s",
"total_time_in_millis" : 39464,
"total_docs" : 281958,
"total_size" : "498.1mb",
"total_size_in_bytes" : 522325012
},
"refresh" : {
"total" : 2526,
"total_time" : "19.1s",
"total_time_in_millis" : 19185
},
"flush" : {
"total" : 2,
"total_time" : "460ms",
"total_time_in_millis" : 460
}
}, {
"routing" : {
"state" : "STARTED",
"primary" : true,
"node" : "qG8xORUDQlCzg6AfEmBe9w",
"relocating_node" : null,
"shard" : 3,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "6.4gb",
"size_in_bytes" : 6949773667
},
"translog" : {
"id" : 1335913333332,
"operations" : 1669
},
"docs" : {
"num_docs" : 4179780,
"max_doc" : 4184705,
"deleted_docs" : 4925
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 13542,
"total_time" : "31.3m",
"total_time_in_millis" : 1879555,
"total_docs" : 15151756,
"total_size" : "25.1gb",
"total_size_in_bytes" : 26956215116
},
"refresh" : {
"total" : 121803,
"total_time" : "21.6m",
"total_time_in_millis" : 1301540
},
"flush" : {
"total" : 130,
"total_time" : "22.2s",
"total_time_in_millis" : 22234
}
} ],
"4" : [ {
"routing" : {
"state" : "STARTED",
"primary" : true,
"node" : "_GNgOXhGRJ-I0c_aZTP1Fg",
"relocating_node" : null,
"shard" : 4,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "6.4gb",
"size_in_bytes" : 6923704094
},
"translog" : {
"id" : 1335913333347,
"operations" : 600
},
"docs" : {
"num_docs" : 4183838,
"max_doc" : 4190239,
"deleted_docs" : 6401
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 13181,
"total_time" : "29.6m",
"total_time_in_millis" : 1777902,
"total_docs" : 14625472,
"total_size" : "24.2gb",
"total_size_in_bytes" : 26068689802
},
"refresh" : {
"total" : 118568,
"total_time" : "18.6m",
"total_time_in_millis" : 1118381
},
"flush" : {
"total" : 127,
"total_time" : "19.9s",
"total_time_in_millis" : 19994
}
}, {
"routing" : {
"state" : "STARTED",
"primary" : false,
"node" : "qG8xORUDQlCzg6AfEmBe9w",
"relocating_node" : null,
"shard" : 4,
"index" : "graylog2"
},
"state" : "STARTED",
"index" : {
"size" : "6.4gb",
"size_in_bytes" : 6922954737
},
"translog" : {
"id" : 1335913333346,
"operations" : 989
},
"docs" : {
"num_docs" : 4183838,
"max_doc" : 4190239,
"deleted_docs" : 6401
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 904,
"total_time" : "1.9m",
"total_time_in_millis" : 117517,
"total_docs" : 666720,
"total_size" : "1.1gb",
"total_size_in_bytes" : 1244906318
},
"refresh" : {
"total" : 8130,
"total_time" : "1.4m",
"total_time_in_millis" : 89254
},
"flush" : {
"total" : 12,
"total_time" : "1.6s",
"total_time_in_millis" : 1648
}
} ]
}
}
}
}{
"cluster_name" : "logger-rdc-elasticsearch",
"nodes" : {
"4-KejzGOTTmj0KblsBVQmw" : {
"name" : "logger-rdc-3",
"transport_address" : "inet[/192.168.4.13:9300]",
"hostname" : "logger-rdc-3",
"indices" : {
"store" : {
"size" : "15.5gb",
"size_in_bytes" : 16680171925
},
"docs" : {
"count" : 8321881,
"deleted" : 1647408
},
"indexing" : {
"index_total" : 237587,
"index_time" : "5.1m",
"index_time_in_millis" : 310186,
"index_current" : 0,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 3,
"time" : "3ms",
"time_in_millis" : 3,
"exists_total" : 3,
"exists_time" : "3ms",
"exists_time_in_millis" : 3,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 18,
"query_time" : "10.3s",
"query_time_in_millis" : 10361,
"query_current" : 0,
"fetch_total" : 18,
"fetch_time" : "59ms",
"fetch_time_in_millis" : 59,
"fetch_current" : 0
},
"cache" : {
"field_evictions" : 0,
"field_size" : "34.8mb",
"field_size_in_bytes" : 36551964,
"filter_count" : 1,
"filter_evictions" : 0,
"filter_size" : "515.2kb",
"filter_size_in_bytes" : 527624
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 8070,
"total_time" : "12.1m",
"total_time_in_millis" : 728853,
"total_docs" : 6245729,
"total_size" : "10.6gb",
"total_size_in_bytes" : 11463719443
},
"refresh" : {
"total" : 144672,
"total_time" : "14.9m",
"total_time_in_millis" : 894328
},
"flush" : {
"total" : 158,
"total_time" : "21.9s",
"total_time_in_millis" : 21921
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment