Skip to content

Instantly share code, notes, and snippets.

@polyfractal

polyfractal/gist:4997040

Created February 20, 2013 16:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save polyfractal/4997040 to your computer and use it in GitHub Desktop.
Save polyfractal/4997040 to your computer and use it in GitHub Desktop.
Download ZIP
Reroute API breaks Cluster/Node/Stats API. More details
Raw
gistfile1.sh
##Listed in order of operations and the resulting output:
## reroute the shards - executed on C1 client node
$curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
"commands":[
{
"move":{
"index":"test",
"shard":2,
"from_node":"J47gdOIwQMq2GTmzzmzJBA",
"to_node":"kH152vsLTL-y20mcLFs9GQ"
}
}
]
}'
{
"ok":true,
"state":{
"master_node":"J47gdOIwQMq2GTmzzmzJBA",
"blocks":{
},
"nodes":{
"ez-rzcnfSESrVYP6zTWksA":{
"name":"S1",
"transport_address":"inet[/144.76.3.102:9300]",
"attributes":{
}
},
"-VNWEiiaSG2IRlwM6kzcxg":{
"name":"C1",
"transport_address":"inet[/144.76.8.228:9300]",
"attributes":{
"data":"false"
}
},
"kH152vsLTL-y20mcLFs9GQ":{
"name":"S3",
"transport_address":"inet[/144.76.2.205:9300]",
"attributes":{
}
},
"J47gdOIwQMq2GTmzzmzJBA":{
"name":"S2",
"transport_address":"inet[/144.76.3.103:9300]",
"attributes":{
}
}
},
"routing_table":{
"indices":{
"test":{
"shards":{
"0":[
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":0,
"index":"test"
}
],
"1":[
{
"state":"STARTED",
"primary":true,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":1,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":1,
"index":"test"
}
],
"2":[
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":2,
"index":"test"
},
{
"state":"RELOCATING",
"primary":true,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":"kH152vsLTL-y20mcLFs9GQ",
"shard":2,
"index":"test"
}
]
}
},
"test123":{
"shards":{
"0":[
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":0,
"index":"test123"
},
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test123"
}
]
}
}
}
},
"routing_nodes":{
"unassigned":[
],
"nodes":{
"ez-rzcnfSESrVYP6zTWksA":[
{
"state":"STARTED",
"primary":true,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":1,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":2,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"ez-rzcnfSESrVYP6zTWksA",
"relocating_node":null,
"shard":0,
"index":"test123"
}
],
"kH152vsLTL-y20mcLFs9GQ":[
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test"
},
{
"state":"INITIALIZING",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":"J47gdOIwQMq2GTmzzmzJBA",
"shard":2,
"index":"test"
},
{
"state":"STARTED",
"primary":true,
"node":"kH152vsLTL-y20mcLFs9GQ",
"relocating_node":null,
"shard":0,
"index":"test123"
}
],
"J47gdOIwQMq2GTmzzmzJBA":[
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":0,
"index":"test"
},
{
"state":"STARTED",
"primary":false,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":null,
"shard":1,
"index":"test"
},
{
"state":"RELOCATING",
"primary":true,
"node":"J47gdOIwQMq2GTmzzmzJBA",
"relocating_node":"kH152vsLTL-y20mcLFs9GQ",
"shard":2,
"index":"test"
}
]
}
},
"allocations":[
]
}
}
## Check API stats - executed quickly after reroute API
## Executed on C1 and S1 just to be sure
$ curl -XGET localhost:9200/_cluster/nodes/stats?all=true
{
"error": "ArithmeticException[Value cannot fit in an int: -2562047788015]",
"status": 500
}
## Running curl -s -XGET 'http://localhost:9200/_nodes/_local/stats?all=true' on all nodes
## Ran while the shard was relocating and the error message was appearing.
## Performed in parallel with Ansible, hence the ansible formatting
polyfractal@Debian-60-squeeze-64-minimal:~/ansible$ ansible es_cluster -u root -m raw -a "curl -s -XGET 'http://localhost:9200/_nodes/_local/stats?all=true&pretty'"
144.76.3.103 | success | rc=0 >>
{
"cluster_name" : "elasticsearch_benchmark",
"nodes" : {
"J47gdOIwQMq2GTmzzmzJBA" : {
"timestamp" : 1361378828241,
"name" : "S2",
"transport_address" : "inet[/144.76.3.103:9300]",
"hostname" : "Debian-60-squeeze-64-minimal",
"indices" : {
"store" : {
"size" : "40.7gb",
"size_in_bytes" : 43807647857,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"docs" : {
"count" : 9999900,
"deleted" : 0
},
"indexing" : {
"index_total" : 113768216,
"index_time" : "140.1d",
"index_time_in_millis" : 12107123757,
"index_current" : 2003,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 1,
"time" : "2ms",
"time_in_millis" : 2,
"exists_total" : 1,
"exists_time" : "2ms",
"exists_time_in_millis" : 2,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 10008324,
"query_time" : "7.3d",
"query_time_in_millis" : 631980281,
"query_current" : 0,
"fetch_total" : 3330672,
"fetch_time" : "48.2m",
"fetch_time_in_millis" : 2897513,
"fetch_current" : 0
},
"cache" : {
"field_evictions" : 0,
"field_size" : "0b",
"field_size_in_bytes" : 0,
"filter_count" : 0,
"filter_evictions" : 0,
"filter_size" : "-418936b",
"filter_size_in_bytes" : -418936,
"bloom_size" : "11.8mb",
"bloom_size_in_bytes" : 12458464,
"id_cache_size" : "0b",
"id_cache_size_in_bytes" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size" : "0b",
"current_size_in_bytes" : 0,
"total" : 6942,
"total_time" : "3.3h",
"total_time_in_millis" : 11942103,
"total_docs" : 176159645,
"total_size" : "491.1gb",
"total_size_in_bytes" : 527413889942
},
"refresh" : {
"total" : 49917,
"total_time" : "5.5h",
"total_time_in_millis" : 19889282
},
"flush" : {
"total" : 12197,
"total_time" : "9.8h",
"total_time_in_millis" : 35416826
}
},
"os" : {
"timestamp" : 1361378828241,
"uptime" : "643 hours, 2 minutes and 19 seconds",
"uptime_in_millis" : 2314939000,
"load_average" : [ 0.06, 0.17, 2.17 ],
"cpu" : {
"sys" : 2,
"user" : 15,
"idle" : 78
},
"mem" : {
"free" : "255.9mb",
"free_in_bytes" : 268345344,
"used" : "30.9gb",
"used_in_bytes" : 33225302016,
"free_percent" : 49,
"used_percent" : 50,
"actual_free" : "15.3gb",
"actual_free_in_bytes" : 16495669248,
"actual_used" : "15.8gb",
"actual_used_in_bytes" : 16997978112
},
"swap" : {
"used" : "160.7mb",
"used_in_bytes" : 168534016,
"free" : "3.8gb",
"free_in_bytes" : 4125364224
}
},
"process" : {
"timestamp" : 1361378828241,
"open_file_descriptors" : 809,
"cpu" : {
"percent" : 155,
"sys" : "3 hours, 26 minutes, 8 seconds and 400 milliseconds",
"sys_in_millis" : 12368400,
"user" : "53 hours, 1 minute, 56 seconds and 480 milliseconds",
"user_in_millis" : 190916480,
"total" : "56 hours, 28 minutes, 4 seconds and 880 milliseconds",
"total_in_millis" : 203284880
},
"mem" : {
"resident" : "15gb",
"resident_in_bytes" : 16106811392,
"share" : "5.6mb",
"share_in_bytes" : 5898240,
"total_virtual" : "18.1gb",
"total_virtual_in_bytes" : 19476443136
}
},
"jvm" : {
"timestamp" : 1361378828242,
"uptime" : "595 hours, 1 minute, 17 seconds and 759 milliseconds",
"uptime_in_millis" : 2142077759,
"mem" : {
"heap_used" : "11.2gb",
"heap_used_in_bytes" : 12122468248,
"heap_committed" : "15.9gb",
"heap_committed_in_bytes" : 17110138880,
"non_heap_used" : "62.2mb",
"non_heap_used_in_bytes" : 65318680,
"non_heap_committed" : "90mb",
"non_heap_committed_in_bytes" : 94392320,
"pools" : {
"Code Cache" : {
"used" : "20.6mb",
"used_in_bytes" : 21648064,
"max" : "48mb",
"max_in_bytes" : 50331648,
"peak_used" : "20.6mb",
"peak_used_in_bytes" : 21688512,
"peak_max" : "48mb",
"peak_max_in_bytes" : 50331648
},
"Par Eden Space" : {
"used" : "396.4mb",
"used_in_bytes" : 415742080,
"max" : "532.5mb",
"max_in_bytes" : 558432256,
"peak_used" : "532.5mb",
"peak_used_in_bytes" : 558432256,
"peak_max" : "532.5mb",
"peak_max_in_bytes" : 558432256
},
"Par Survivor Space" : {
"used" : "3.2mb",
"used_in_bytes" : 3434896,
"max" : "66.5mb",
"max_in_bytes" : 69730304,
"peak_used" : "66.5mb",
"peak_used_in_bytes" : 69730304,
"peak_max" : "66.5mb",
"peak_max_in_bytes" : 69730304
},
"CMS Old Gen" : {
"used" : "10.8gb",
"used_in_bytes" : 11703291272,
"max" : "15.3gb",
"max_in_bytes" : 16481976320,
"peak_used" : "11.5gb",
"peak_used_in_bytes" : 12368076416,
"peak_max" : "15.3gb",
"peak_max_in_bytes" : 16481976320
},
"CMS Perm Gen" : {
"used" : "41.6mb",
"used_in_bytes" : 43670616,
"max" : "82mb",
"max_in_bytes" : 85983232,
"peak_used" : "41.6mb",
"peak_used_in_bytes" : 43670616,
"peak_max" : "82mb",
"peak_max_in_bytes" : 85983232
}
}
},
"threads" : {
"count" : 81,
"peak_count" : 1884
},
"gc" : {
"collection_count" : 19102,
"collection_time" : "12 minutes, 54 seconds and 59 milliseconds",
"collection_time_in_millis" : 774059,
"collectors" : {
"ParNew" : {
"collection_count" : 19098,
"collection_time" : "12 minutes, 53 seconds and 246 milliseconds",
"collection_time_in_millis" : 773246
},
"ConcurrentMarkSweep" : {
"collection_count" : 4,
"collection_time" : "813 milliseconds",
"collection_time_in_millis" : 813
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 2379,
"used" : "141.9mb",
"used_in_bytes" : 148831994,
"total_capacity" : "141.9mb",
"total_capacity_in_bytes" : 148831994
},
"mapped" : {
"count" : 0,
"used" : "0b",
"used_in_bytes" : 0,
"total_capacity" : "0b",
"total_capacity_in_bytes" : 0
}
}
},
"thread_pool" : {
"generic" : {
"threads" : 2,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 0,
"completed" : 871104
},
"index" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 113342179
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 12210
},
"cache" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 117372
},
"get" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 1
},
"search" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 13235485
},
"snapshot" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 13174
},
"merge" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 54310
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 49795
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"largest" : -1,
"completed" : 4907581
},
"bulk" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 8358
}
},
"network" : {
"tcp" : {
"active_opens" : 925,
"passive_opens" : 60644,
"curr_estab" : 79,
"in_segs" : 922723258,
"out_segs" : 1051507108,
"retrans_segs" : 153113,
"estab_resets" : 186,
"attempt_fails" : 438,
"in_errs" : 202,
"out_rsts" : 22280
}
},
"fs" : {
"timestamp" : 1361378828242,
"data" : [ {
"path" : "/var/lib/elasticsearch/elasticsearch_benchmark/nodes/0",
"mount" : "/",
"dev" : "/dev/md2",
"total" : "1015.9gb",
"total_in_bytes" : 1090848337920,
"free" : "969.2gb",
"free_in_bytes" : 1040711577600,
"available" : "918gb",
"available_in_bytes" : 985736056832,
"disk_reads" : 6709955,
"disk_writes" : 685277272,
"disk_read_size" : "247.9gb",
"disk_read_size_in_bytes" : 266215659520,
"disk_write_size" : "2613.6gb",
"disk_write_size_in_bytes" : 2806365339648,
"disk_queue" : "0",
"disk_service_time" : "0"
} ]
},
"transport" : {
"server_open" : 36,
"rx_count" : 182926385,
"rx_size" : "138.1gb",
"rx_size_in_bytes" : 148337613933,
"tx_count" : 157961709,
"tx_size" : "162gb",
"tx_size_in_bytes" : 173972714666
},
"http" : {
"current_open" : 3,
"total_opened" : 4378
}
}
}
}
144.76.3.102 | success | rc=0 >>
{
"cluster_name" : "elasticsearch_benchmark",
"nodes" : {
"ez-rzcnfSESrVYP6zTWksA" : {
"timestamp" : 1361378828251,
"name" : "S1",
"transport_address" : "inet[/144.76.3.102:9300]",
"hostname" : "Debian-60-squeeze-64-minimal",
"indices" : {
"store" : {
"size" : "27.2gb",
"size_in_bytes" : 29207177550,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"docs" : {
"count" : 6667075,
"deleted" : 0
},
"indexing" : {
"index_total" : 156881055,
"index_time" : "125.4d",
"index_time_in_millis" : 10841797463,
"index_current" : 3168,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 0,
"time" : "0s",
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time" : "0s",
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 144519,
"query_time" : "1.5h",
"query_time_in_millis" : 5654687,
"query_current" : 0,
"fetch_total" : 112114,
"fetch_time" : "4.5s",
"fetch_time_in_millis" : 4590,
"fetch_current" : 0
},
"cache" : {
"field_evictions" : 0,
"field_size" : "0b",
"field_size_in_bytes" : 0,
"filter_count" : -1,
"filter_evictions" : 0,
"filter_size" : "-834128b",
"filter_size_in_bytes" : -834128,
"bloom_size" : "5.9mb",
"bloom_size_in_bytes" : 6250768,
"id_cache_size" : "0b",
"id_cache_size_in_bytes" : 0
},
"merges" : {
"current" : 7,
"current_docs" : 17917691,
"current_size" : "19.8gb",
"current_size_in_bytes" : 21282790288,
"total" : 4652,
"total_time" : "3.4h",
"total_time_in_millis" : 12246838,
"total_docs" : 312656615,
"total_size" : "611.6gb",
"total_size_in_bytes" : 656802646552
},
"refresh" : {
"total" : 34371,
"total_time" : "3.3h",
"total_time_in_millis" : 11935634
},
"flush" : {
"total" : 5790,
"total_time" : "7.5h",
"total_time_in_millis" : 27212611
}
},
"os" : {
"timestamp" : 1361378828251,
"uptime" : "641 hours, 20 minutes and 34 seconds",
"uptime_in_millis" : 2308834000,
"load_average" : [ 0.02, 0.04, 0.16 ],
"cpu" : {
"sys" : 0,
"user" : 0,
"idle" : 97
},
"mem" : {
"free" : "194.1mb",
"free_in_bytes" : 203608064,
"used" : "31gb",
"used_in_bytes" : 33290039296,
"free_percent" : 43,
"used_percent" : 56,
"actual_free" : "13.5gb",
"actual_free_in_bytes" : 14563196928,
"actual_used" : "17.6gb",
"actual_used_in_bytes" : 18930450432
},
"swap" : {
"used" : "178.3mb",
"used_in_bytes" : 187031552,
"free" : "3.8gb",
"free_in_bytes" : 4106866688
}
},
"process" : {
"timestamp" : 1361378828252,
"open_file_descriptors" : 621,
"cpu" : {
"percent" : 0,
"sys" : "3 hours, 34 minutes, 34 seconds and 920 milliseconds",
"sys_in_millis" : 12874920,
"user" : "30 hours, 35 minutes, 53 seconds and 940 milliseconds",
"user_in_millis" : 110153940,
"total" : "34 hours, 10 minutes, 28 seconds and 860 milliseconds",
"total_in_millis" : 123028860
},
"mem" : {
"resident" : "16.9gb",
"resident_in_bytes" : 18203938816,
"share" : "5.3mb",
"share_in_bytes" : 5623808,
"total_virtual" : "17.3gb",
"total_virtual_in_bytes" : 18643701760
}
},
"jvm" : {
"timestamp" : 1361378828252,
"uptime" : "160 hours, 37 minutes, 33 seconds and 722 milliseconds",
"uptime_in_millis" : 578253722,
"mem" : {
"heap_used" : "8.1gb",
"heap_used_in_bytes" : 8735667352,
"heap_committed" : "15.9gb",
"heap_committed_in_bytes" : 17110138880,
"non_heap_used" : "59.5mb",
"non_heap_used_in_bytes" : 62454432,
"non_heap_committed" : "86.8mb",
"non_heap_committed_in_bytes" : 91029504,
"pools" : {
"Code Cache" : {
"used" : "18.7mb",
"used_in_bytes" : 19685824,
"max" : "48mb",
"max_in_bytes" : 50331648,
"peak_used" : "18.8mb",
"peak_used_in_bytes" : 19777408,
"peak_max" : "48mb",
"peak_max_in_bytes" : 50331648
},
"Par Eden Space" : {
"used" : "361.3mb",
"used_in_bytes" : 378866504,
"max" : "532.5mb",
"max_in_bytes" : 558432256,
"peak_used" : "532.5mb",
"peak_used_in_bytes" : 558432256,
"peak_max" : "532.5mb",
"peak_max_in_bytes" : 558432256
},
"Par Survivor Space" : {
"used" : "3.1mb",
"used_in_bytes" : 3267968,
"max" : "66.5mb",
"max_in_bytes" : 69730304,
"peak_used" : "66.5mb",
"peak_used_in_bytes" : 69730304,
"peak_max" : "66.5mb",
"peak_max_in_bytes" : 69730304
},
"CMS Old Gen" : {
"used" : "7.7gb",
"used_in_bytes" : 8353532880,
"max" : "15.3gb",
"max_in_bytes" : 16481976320,
"peak_used" : "15.3gb",
"peak_used_in_bytes" : 16459831872,
"peak_max" : "15.3gb",
"peak_max_in_bytes" : 16481976320
},
"CMS Perm Gen" : {
"used" : "40.7mb",
"used_in_bytes" : 42768608,
"max" : "82mb",
"max_in_bytes" : 85983232,
"peak_used" : "40.7mb",
"peak_used_in_bytes" : 42768608,
"peak_max" : "82mb",
"peak_max_in_bytes" : 85983232
}
}
},
"threads" : {
"count" : 77,
"peak_count" : 1884
},
"gc" : {
"collection_count" : 15317,
"collection_time" : "13 minutes, 11 seconds and 740 milliseconds",
"collection_time_in_millis" : 791740,
"collectors" : {
"ParNew" : {
"collection_count" : 15203,
"collection_time" : "12 minutes, 37 seconds and 24 milliseconds",
"collection_time_in_millis" : 757024
},
"ConcurrentMarkSweep" : {
"collection_count" : 114,
"collection_time" : "34 seconds and 716 milliseconds",
"collection_time_in_millis" : 34716
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 2010,
"used" : "134mb",
"used_in_bytes" : 140594624,
"total_capacity" : "134mb",
"total_capacity_in_bytes" : 140594624
},
"mapped" : {
"count" : 0,
"used" : "0b",
"used_in_bytes" : 0,
"total_capacity" : "0b",
"total_capacity_in_bytes" : 0
}
}
},
"thread_pool" : {
"generic" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 256268
},
"index" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 156850395
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 5861
},
"cache" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 60041
},
"get" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"search" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 195127
},
"snapshot" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 8411
},
"merge" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 37295
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 34264
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"largest" : -1,
"completed" : 1568233
},
"bulk" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 4364
}
},
"network" : {
"tcp" : {
"active_opens" : 873,
"passive_opens" : 193538,
"curr_estab" : 86,
"in_segs" : 2026668822,
"out_segs" : 2090425545,
"retrans_segs" : 173838,
"estab_resets" : 5088,
"attempt_fails" : 450,
"in_errs" : 242,
"out_rsts" : 18050
}
},
"fs" : {
"timestamp" : 1361378828253,
"data" : [ {
"path" : "/var/lib/elasticsearch/elasticsearch_benchmark/nodes/0",
"mount" : "/",
"dev" : "/dev/md2",
"total" : "1015.9gb",
"total_in_bytes" : 1090848337920,
"free" : "984gb",
"free_in_bytes" : 1056649801728,
"available" : "932.8gb",
"available_in_bytes" : 1001674280960,
"disk_reads" : 5733548,
"disk_writes" : 737387651,
"disk_read_size" : "194.6gb",
"disk_read_size_in_bytes" : 209036359680,
"disk_write_size" : "2812.4gb",
"disk_write_size_in_bytes" : 3019828723712,
"disk_queue" : "0",
"disk_service_time" : "0"
} ]
},
144.76.2.205 | success | rc=0 >>
{
"cluster_name" : "elasticsearch_benchmark",
"nodes" : {
"kH152vsLTL-y20mcLFs9GQ" : {
"timestamp" : 1361378828252,
"name" : "S3",
"transport_address" : "inet[/144.76.2.205:9300]",
"hostname" : "Debian-60-squeeze-64-minimal",
"indices" : {
"store" : {
"size" : "16.5gb",
"size_in_bytes" : 17750418910,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"docs" : {
"count" : 3332825,
"deleted" : 0
},
"indexing" : {
"index_total" : 109079109,
"index_time" : "74.3d",
"index_time_in_millis" : 6425954679,
"index_current" : 52535,
"delete_total" : 0,
"delete_time" : "0s",
"delete_time_in_millis" : 0,
"delete_current" : 0
},
"get" : {
"total" : 2,
"time" : "2ms",
"time_in_millis" : 2,
"exists_total" : 1,
"exists_time" : "2ms",
"exists_time_in_millis" : 2,
"missing_total" : 1,
"missing_time" : "0s",
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"query_total" : 9837062,
"query_time" : "3.1d",
"query_time_in_millis" : 274939193,
"query_current" : 0,
"fetch_total" : 3442003,
"fetch_time" : "25.1m",
"fetch_time_in_millis" : 1511150,
"fetch_current" : 0
},
"cache" : {
"field_evictions" : 0,
"field_size" : "0b",
"field_size_in_bytes" : 0,
"filter_count" : 0,
"filter_evictions" : 0,
"filter_size" : "-834192b",
"filter_size_in_bytes" : -834192,
"bloom_size" : "0b",
"bloom_size_in_bytes" : 0,
"id_cache_size" : "0b",
"id_cache_size_in_bytes" : 0
},
"merges" : {
"current" : 1,
"current_docs" : 19179,
"current_size" : "87.5mb",
"current_size_in_bytes" : 91763722,
"total" : 4723,
"total_time" : "1.9h",
"total_time_in_millis" : 7156548,
"total_docs" : 188705565,
"total_size" : "328.1gb",
"total_size_in_bytes" : 352332073622
},
"refresh" : {
"total" : 35642,
"total_time" : "4h",
"total_time_in_millis" : 14487592
},
"flush" : {
"total" : 11468,
"total_time" : "6.5h",
"total_time_in_millis" : 23574420
}
},
"os" : {
"timestamp" : 1361378828253,
"uptime" : "641 hours, 12 minutes and 22 seconds",
"uptime_in_millis" : 2308342000,
"load_average" : [ 0.42, 0.16, 0.5 ],
"cpu" : {
"sys" : 3,
"user" : 9,
"idle" : 82
},
"mem" : {
"free" : "5.6gb",
"free_in_bytes" : 6099591168,
"used" : "25.5gb",
"used_in_bytes" : 27394056192,
"free_percent" : 54,
"used_percent" : 45,
"actual_free" : "16.9gb",
"actual_free_in_bytes" : 18164064256,
"actual_used" : "14.2gb",
"actual_used_in_bytes" : 15329583104
},
"swap" : {
"used" : "13.5mb",
"used_in_bytes" : 14209024,
"free" : "3.9gb",
"free_in_bytes" : 4279689216
}
},
"process" : {
"timestamp" : 1361378828253,
"open_file_descriptors" : 439,
"cpu" : {
"percent" : 111,
"sys" : "2 hours, 41 minutes, 32 seconds and 310 milliseconds",
"sys_in_millis" : 9692310,
"user" : "36 hours, 9 minutes, 59 seconds and 820 milliseconds",
"user_in_millis" : 130199820,
"total" : "38 hours, 51 minutes, 32 seconds and 130 milliseconds",
"total_in_millis" : 139892130
},
"mem" : {
"resident" : "13.5gb",
"resident_in_bytes" : 14519554048,
"share" : "6mb",
"share_in_bytes" : 6332416,
"total_virtual" : "17.7gb",
"total_virtual_in_bytes" : 19038695424
}
},
"jvm" : {
"timestamp" : 1361378828253,
"uptime" : "594 hours, 33 minutes, 44 seconds and 331 milliseconds",
"uptime_in_millis" : 2140424331,
"mem" : {
"heap_used" : "7.8gb",
"heap_used_in_bytes" : 8469861056,
"heap_committed" : "15.9gb",
"heap_committed_in_bytes" : 17110138880,
"non_heap_used" : "60.1mb",
"non_heap_used_in_bytes" : 63109280,
"non_heap_committed" : "86.9mb",
"non_heap_committed_in_bytes" : 91156480,
"pools" : {
"Code Cache" : {
"used" : "19.4mb",
"used_in_bytes" : 20395264,
"max" : "48mb",
"max_in_bytes" : 50331648,
"peak_used" : "19.4mb",
"peak_used_in_bytes" : 20406016,
"peak_max" : "48mb",
"peak_max_in_bytes" : 50331648
},
"Par Eden Space" : {
"used" : "425mb",
"used_in_bytes" : 445647592,
"max" : "532.5mb",
"max_in_bytes" : 558432256,
"peak_used" : "532.5mb",
"peak_used_in_bytes" : 558432256,
"peak_max" : "532.5mb",
"peak_max_in_bytes" : 558432256
},
"Par Survivor Space" : {
"used" : "1.4mb",
"used_in_bytes" : 1492456,
"max" : "66.5mb",
"max_in_bytes" : 69730304,
"peak_used" : "66.5mb",
"peak_used_in_bytes" : 69730304,
"peak_max" : "66.5mb",
"peak_max_in_bytes" : 69730304
},
"CMS Old Gen" : {
"used" : "7.4gb",
"used_in_bytes" : 8022723080,
"max" : "15.3gb",
"max_in_bytes" : 16481976320,
"peak_used" : "11.5gb",
"peak_used_in_bytes" : 12365203168,
"peak_max" : "15.3gb",
"peak_max_in_bytes" : 16481976320
},
"CMS Perm Gen" : {
"used" : "40.7mb",
"used_in_bytes" : 42714016,
"max" : "82mb",
"max_in_bytes" : 85983232,
"peak_used" : "40.7mb",
"peak_used_in_bytes" : 42714016,
"peak_max" : "82mb",
"peak_max_in_bytes" : 85983232
}
}
},
"threads" : {
"count" : 80,
"peak_count" : 1883
},
"gc" : {
"collection_count" : 11614,
"collection_time" : "8 minutes, 18 seconds and 107 milliseconds",
"collection_time_in_millis" : 498107,
"collectors" : {
"ParNew" : {
"collection_count" : 11612,
"collection_time" : "8 minutes, 17 seconds and 412 milliseconds",
"collection_time_in_millis" : 497412
},
"ConcurrentMarkSweep" : {
"collection_count" : 2,
"collection_time" : "695 milliseconds",
"collection_time_in_millis" : 695
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 2768,
"used" : "169.9mb",
"used_in_bytes" : 178231097,
"total_capacity" : "169.9mb",
"total_capacity_in_bytes" : 178231097
},
"mapped" : {
"count" : 0,
"used" : "0b",
"used_in_bytes" : 0,
"total_capacity" : "0b",
"total_capacity_in_bytes" : 0
}
}
},
"thread_pool" : {
"generic" : {
"threads" : 4,
"queue" : 0,
"active" : 2,
"rejected" : 0,
"largest" : 0,
"completed" : 454456
},
"index" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 109099469
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 11480
},
"cache" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 78112
},
"get" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 5
},
"search" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 13308545
},
"snapshot" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 10045
},
"merge" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 42342
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"largest" : -1,
"completed" : 35451
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"largest" : -1,
"completed" : 4943344
},
"bulk" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 4364
}
},
"network" : {
"tcp" : {
"active_opens" : 21642,
"passive_opens" : 87901,
"curr_estab" : 79,
"in_segs" : 394139093,
"out_segs" : 417270877,
"retrans_segs" : 28062,
"estab_resets" : 177,
"attempt_fails" : 370,
"in_errs" : 75,
"out_rsts" : 6892
}
},
"fs" : {
"timestamp" : 1361378828254,
"data" : [ {
"path" : "/var/lib/elasticsearch/elasticsearch_benchmark/nodes/0",
"mount" : "/",
"dev" : "/dev/md2",
"total" : "1015.9gb",
"total_in_bytes" : 1090848337920,
"free" : "994.7gb",
"free_in_bytes" : 1068083478528,
"available" : "943.5gb",
"available_in_bytes" : 1013107957760,
"disk_reads" : 380878,
"disk_writes" : 331184225,
"disk_read_size" : "20.4gb",
"disk_read_ "transport" : {
"server_open" : 36,
"rx_count" : 151218350,
"rx_size" : "144.3gb",
"rx_size_in_bytes" : 155045029541,
size_in_bytes" : 21938227200,
"tx_count" : 125459795,
"disk_write_size" : "1263gb",
"tx_size" : "93.5gb",
"disk_write_size_in_bytes" : 1356241309696,
"tx_size_in_bytes" : 100429683098
"disk_queue" : "0",
},
"disk_service_time" : "0"
"http" : {
} ]
"current_open" : 9,
},
"total_opened" : 24474
"transport" : {
}
"server_open" : 36,
}
"rx_count" : 167290341,
}
"rx_size" : "155.9gb",
}
"rx_size_in_bytes" : 167495293626,
"tx_count" : 142811276,
"tx_size" : "70.6gb",
"tx_size_in_bytes" : 75859638076
},
"http" : {
"current_open" : 3,
"total_opened" : 6
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment