Skip to content

Instantly share code, notes, and snippets.

@johnc10uk
Created October 28, 2014 11:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save johnc10uk/441b58a6181bc07af4c1 to your computer and use it in GitHub Desktop.
Save johnc10uk/441b58a6181bc07af4c1 to your computer and use it in GitHub Desktop.
elasticsearch info
{"cluster_name":"elasticsearch","nodes":{"kxyKdBB5Ri6AIx-gexM-NQ":{"name":"elasticsearch2","transport_address":"inet[/10.20.149.92:9300]","host":"elasticsearch2","ip":"10.20.149.92","version":"1.3.4","build":"a70f3cc","http_address":"inet[/10.20.149.92:9200]","attributes":{"aws_availability_zone":"us-west-2b","max_local_storage_nodes":"1"},"settings":{"index":{"mapper":{"dynamic":"true"},"number_of_replicas":"2","number_of_shards":"5","refresh_interval":"30s"},"bootstrap":{"mlockall":"true"},"gateway":{"expected_nodes":"1"},"pidfile":"/usr/local/var/run/elasticsearch2.pid","plugin":{"mandatory":"cloud-aws"},"node":{"max_local_storage_nodes":"1","name":"elasticsearch2"},"http":{"port":"9200"},"name":"elasticsearch2","action":{"auto_create_index":"true","disable_delete_all_indices":"true"},"path":{"data":"/vol/data1,/vol/data2,/vol/data3,/vol/data4","home":"/usr/local/elasticsearch","conf":"/usr/local/etc/elasticsearch","logs":"/usr/local/var/log/elasticsearch"},"cloud":{"aws":{"region":"us-west-2"},"node":{"auto_attributes":"true"}},"config":"/usr/local/etc/elasticsearch/elasticsearch.yml","cluster":{"name":"elasticsearch"},"discovery":{"type":"ec2","zen":{"minimum_master_nodes":"1","ping":{"multicast":{"enabled":"true"}}},"ec2":{"groups":"prod-us1-elasticsearch"}},"foreground":"yes"},"os":{"refresh_interval_in_millis":1000,"available_processors":2,"cpu":{"vendor":"Intel","model":"Xeon","mhz":2494,"total_cores":2,"total_sockets":1,"cores_per_socket":2,"cache_size_in_bytes":25600},"mem":{"total_in_bytes":15775350784},"swap":{"total_in_bytes":0}},"process":{"refresh_interval_in_millis":1000,"id":8198,"max_file_descriptors":64000,"mlockall":true},"jvm":{"pid":8198,"version":"1.7.0_55","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"24.55-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1414151301201,"mem":{"heap_init_in_bytes":9464446976,"heap_max_in_bytes":9447014400,"non_heap_init_in_bytes":24313856,"non_heap_max_in_bytes":136314880,"direct_max_in_bytes":9447014400},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Par Eden Space","Par Survivor Space","CMS Old Gen","CMS Perm Gen"]},"thread_pool":{"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"index":{"type":"fixed","min":2,"max":2,"queue_size":"200"},"snapshot_data":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"bench":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"snapshot":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"merge":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"suggest":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"bulk":{"type":"fixed","min":2,"max":2,"queue_size":"50"},"optimize":{"type":"fixed","min":1,"max":1,"queue_size":-1},"warmer":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"flush":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":6,"max":6,"queue_size":"1k"},"percolate":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"refresh":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1}},"network":{"refresh_interval_in_millis":5000,"primary_interface":{"address":"10.20.149.92","name":"eth0","mac_address":"06:19:88:58:57:90"}},"transport":{"bound_address":"inet[/0.0.0.0:9300]","publish_address":"inet[/10.20.149.92:9300]"},"http":{"bound_address":"inet[/0.0.0.0:9200]","publish_address":"inet[/10.20.149.92:9200]","max_content_length_in_bytes":104857600},"plugins":[{"name":"cloud-aws","version":"2.3.0","description":"Cloud AWS Plugin","jvm":true,"site":false},{"name":"paramedic","version":"NA","description":"No description found.","url":"/_plugin/paramedic/","jvm":false,"site":true},{"name":"HQ","version":"NA","description":"No description found.","url":"/_plugin/HQ/","jvm":false,"site":true},{"name":"head","version":"NA","description":"No description found.","url":"/_plugin/head/","jvm":false,"site":true},{"name":"bigdesk","version":"NA","description":"No description found.","url":"/_plugin/bigdesk/","jvm":false,"site":true}]},"LNlCcpOwSHafvu7OpO4Xaw":{"name":"elasticsearch1","transport_address":"inet[/10.20.147.248:9300]","host":"elasticsearch1","ip":"10.20.147.248","version":"1.3.4","build":"a70f3cc","http_address":"inet[/10.20.147.248:9200]","attributes":{"aws_availability_zone":"us-west-2a","max_local_storage_nodes":"1"},"settings":{"index":{"mapper":{"dynamic":"true"},"number_of_replicas":"2","number_of_shards":"5","refresh_interval":"30s"},"bootstrap":{"mlockall":"true"},"gateway":{"expected_nodes":"1"},"pidfile":"/usr/local/var/run/elasticsearch1.pid","plugin":{"mandatory":"cloud-aws"},"node":{"max_local_storage_nodes":"1","name":"elasticsearch1"},"http":{"port":"9200"},"name":"elasticsearch1","action":{"auto_create_index":"true","disable_delete_all_indices":"true"},"path":{"data":"/vol/data1,/vol/data2,/vol/data3,/vol/data4","home":"/usr/local/elasticsearch","conf":"/usr/local/etc/elasticsearch","logs":"/usr/local/var/log/elasticsearch"},"cloud":{"aws":{"region":"us-west-2"},"node":{"auto_attributes":"true"}},"config":"/usr/local/etc/elasticsearch/elasticsearch.yml","cluster":{"name":"elasticsearch"},"discovery":{"type":"ec2","zen":{"minimum_master_nodes":"1","ping":{"multicast":{"enabled":"true"}}},"ec2":{"groups":"prod-us1-elasticsearch"}},"foreground":"yes"},"os":{"refresh_interval_in_millis":1000,"available_processors":2,"cpu":{"vendor":"Intel","model":"Xeon","mhz":2494,"total_cores":2,"total_sockets":1,"cores_per_socket":2,"cache_size_in_bytes":25600},"mem":{"total_in_bytes":15775350784},"swap":{"total_in_bytes":0}},"process":{"refresh_interval_in_millis":1000,"id":28077,"max_file_descriptors":64000,"mlockall":true},"jvm":{"pid":28077,"version":"1.7.0_55","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"24.55-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1414151840503,"mem":{"heap_init_in_bytes":9464446976,"heap_max_in_bytes":9447014400,"non_heap_init_in_bytes":24313856,"non_heap_max_in_bytes":136314880,"direct_max_in_bytes":9447014400},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Par Eden Space","Par Survivor Space","CMS Old Gen","CMS Perm Gen"]},"thread_pool":{"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"index":{"type":"fixed","min":2,"max":2,"queue_size":"200"},"snapshot_data":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"bench":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"snapshot":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"merge":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"suggest":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"bulk":{"type":"fixed","min":2,"max":2,"queue_size":"50"},"optimize":{"type":"fixed","min":1,"max":1,"queue_size":-1},"warmer":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"flush":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":6,"max":6,"queue_size":"1k"},"percolate":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"refresh":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1}},"network":{"refresh_interval_in_millis":5000,"primary_interface":{"address":"10.20.147.248","name":"eth0","mac_address":"02:8D:81:E2:F8:CD"}},"transport":{"bound_address":"inet[/0.0.0.0:9300]","publish_address":"inet[/10.20.147.248:9300]"},"http":{"bound_address":"inet[/0.0.0.0:9200]","publish_address":"inet[/10.20.147.248:9200]","max_content_length_in_bytes":104857600},"plugins":[{"name":"cloud-aws","version":"2.3.0","description":"Cloud AWS Plugin","jvm":true,"site":false},{"name":"paramedic","version":"NA","description":"No description found.","url":"/_plugin/paramedic/","jvm":false,"site":true},{"name":"HQ","version":"NA","description":"No description found.","url":"/_plugin/HQ/","jvm":false,"site":true},{"name":"head","version":"NA","description":"No description found.","url":"/_plugin/head/","jvm":false,"site":true},{"name":"bigdesk","version":"NA","description":"No description found.","url":"/_plugin/bigdesk/","jvm":false,"site":true}]},"vs3KTjwtRTycfP-GwaAniw":{"name":"elasticsearch3","transport_address":"inet[/10.20.151.176:9300]","host":"elasticsearch3","ip":"10.20.151.176","version":"1.3.4","build":"a70f3cc","http_address":"inet[/10.20.151.176:9200]","attributes":{"aws_availability_zone":"us-west-2c","max_local_storage_nodes":"1"},"settings":{"index":{"mapper":{"dynamic":"true"},"number_of_replicas":"2","number_of_shards":"5","refresh_interval":"30s"},"bootstrap":{"mlockall":"true"},"gateway":{"expected_nodes":"1"},"pidfile":"/usr/local/var/run/elasticsearch3.pid","plugin":{"mandatory":"cloud-aws"},"node":{"max_local_storage_nodes":"1","name":"elasticsearch3"},"http":{"port":"9200"},"name":"elasticsearch3","action":{"auto_create_index":"true","disable_delete_all_indices":"true"},"path":{"data":"/vol/data1,/vol/data2,/vol/data3,/vol/data4","home":"/usr/local/elasticsearch","conf":"/usr/local/etc/elasticsearch","logs":"/usr/local/var/log/elasticsearch"},"cloud":{"aws":{"region":"us-west-2"},"node":{"auto_attributes":"true"}},"config":"/usr/local/etc/elasticsearch/elasticsearch.yml","cluster":{"name":"elasticsearch"},"discovery":{"type":"ec2","zen":{"minimum_master_nodes":"1","ping":{"multicast":{"enabled":"true"}}},"ec2":{"groups":"prod-us1-elasticsearch"}},"foreground":"yes"},"os":{"refresh_interval_in_millis":1000,"available_processors":2,"cpu":{"vendor":"Intel","model":"Xeon","mhz":2494,"total_cores":2,"total_sockets":1,"cores_per_socket":2,"cache_size_in_bytes":25600},"mem":{"total_in_bytes":15775350784},"swap":{"total_in_bytes":0}},"process":{"refresh_interval_in_millis":1000,"id":12981,"max_file_descriptors":64000,"mlockall":true},"jvm":{"pid":12981,"version":"1.7.0_55","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"24.55-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1414150999927,"mem":{"heap_init_in_bytes":9464446976,"heap_max_in_bytes":9447014400,"non_heap_init_in_bytes":24313856,"non_heap_max_in_bytes":136314880,"direct_max_in_bytes":9447014400},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Par Eden Space","Par Survivor Space","CMS Old Gen","CMS Perm Gen"]},"thread_pool":{"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"index":{"type":"fixed","min":2,"max":2,"queue_size":"200"},"snapshot_data":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"bench":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"snapshot":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"merge":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"suggest":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"bulk":{"type":"fixed","min":2,"max":2,"queue_size":"50"},"optimize":{"type":"fixed","min":1,"max":1,"queue_size":-1},"warmer":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"flush":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":6,"max":6,"queue_size":"1k"},"percolate":{"type":"fixed","min":2,"max":2,"queue_size":"1k"},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"refresh":{"type":"scaling","min":1,"max":1,"keep_alive":"5m","queue_size":-1}},"network":{"refresh_interval_in_millis":5000,"primary_interface":{"address":"10.20.151.176","name":"eth0","mac_address":"0A:FB:A2:77:64:E7"}},"transport":{"bound_address":"inet[/0.0.0.0:9300]","publish_address":"inet[/10.20.151.176:9300]"},"http":{"bound_address":"inet[/0.0.0.0:9200]","publish_address":"inet[/10.20.151.176:9200]","max_content_length_in_bytes":104857600},"plugins":[{"name":"cloud-aws","version":"2.3.0","description":"Cloud AWS Plugin","jvm":true,"site":false},{"name":"paramedic","version":"NA","description":"No description found.","url":"/_plugin/paramedic/","jvm":false,"site":true},{"name":"HQ","version":"NA","description":"No description found.","url":"/_plugin/HQ/","jvm":false,"site":true},{"name":"head","version":"NA","description":"No description found.","url":"/_plugin/head/","jvm":false,"site":true},{"name":"bigdesk","version":"NA","description":"No description found.","url":"/_plugin/bigdesk/","jvm":false,"site":true}]}}}
{"cluster_name":"elasticsearch","nodes":{"kxyKdBB5Ri6AIx-gexM-NQ":{"timestamp":1414493683902,"name":"elasticsearch2","transport_address":"inet[/10.20.149.92:9300]","host":"elasticsearch2","ip":["inet[/10.20.149.92:9300]","NONE"],"attributes":{"aws_availability_zone":"us-west-2b","max_local_storage_nodes":"1"},"indices":{"docs":{"count":65997539,"deleted":12817545},"store":{"size_in_bytes":29096164817,"throttle_time_in_millis":1061},"indexing":{"index_total":10971313,"index_time_in_millis":4022762,"index_current":0,"delete_total":7,"delete_time_in_millis":1,"delete_current":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10108932,"query_time_in_millis":1076005,"query_current":0,"fetch_total":180751,"fetch_time_in_millis":25204,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":25125,"total_time_in_millis":6387095,"total_docs":137689950,"total_size_in_bytes":53348657524},"refresh":{"total":229332,"total_time_in_millis":2993055},"flush":{"total":5106,"total_time_in_millis":279919},"warmer":{"current":0,"total":490472,"total_time_in_millis":49558},"filter_cache":{"memory_size_in_bytes":496479440,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":132573594,"evictions":0,"fields":{"field_updated_l":{"memory_size_in_bytes":36563408},"attr_allcategories_y":{"memory_size_in_bytes":2612345},"attr_hw_pow_s":{"memory_size_in_bytes":59210673},"field_added_l":{"memory_size_in_bytes":34187168}}},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":541,"memory_in_bytes":826144816,"index_writer_memory_in_bytes":5882920,"version_map_memory_in_bytes":65784},"translog":{"operations":55102,"size_in_bytes":0},"suggest":{"total":0,"time_in_millis":0,"current":0}},"os":{"timestamp":1414493684335,"uptime_in_millis":932206,"load_average":[1.29,1.33,1.31],"cpu":{"sys":1,"user":53,"idle":44,"usage":54,"stolen":0},"mem":{"free_in_bytes":200921088,"used_in_bytes":15574429696,"free_percent":29,"used_percent":70,"actual_free_in_bytes":4698820608,"actual_used_in_bytes":11076530176},"swap":{"used_in_bytes":0,"free_in_bytes":0}},"process":{"timestamp":1414493684336,"open_file_descriptors":2178,"cpu":{"percent":110,"sys_in_millis":2871990,"user_in_millis":76491300,"total_in_millis":79363290},"mem":{"resident_in_bytes":10449211392,"share_in_bytes":492613632,"total_virtual_in_bytes":22880407552}},"jvm":{"timestamp":1414493684337,"uptime_in_millis":342383136,"mem":{"heap_used_in_bytes":7340067872,"heap_used_percent":77,"heap_committed_in_bytes":9447014400,"heap_max_in_bytes":9447014400,"non_heap_used_in_bytes":67045384,"non_heap_committed_in_bytes":100466688,"pools":{"young":{"used_in_bytes":49293480,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":12736208,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":7278038184,"max_in_bytes":9289990144,"peak_used_in_bytes":7520877624,"peak_max_in_bytes":9289990144}}},"threads":{"count":50,"peak_count":58},"gc":{"collectors":{"young":{"collection_count":52465,"collection_time_in_millis":3234018},"old":{"collection_count":1270,"collection_time_in_millis":160779}}},"buffer_pools":{"direct":{"count":2541,"used_in_bytes":48004217,"total_capacity_in_bytes":48004217},"mapped":{"count":634,"used_in_bytes":11939397791,"total_capacity_in_bytes":11939397791}}},"thread_pool":{"generic":{"threads":5,"queue":0,"active":0,"rejected":0,"largest":12,"completed":2579203},"index":{"threads":2,"queue":0,"active":0,"rejected":0,"largest":2,"completed":10971371},"snapshot_data":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bench":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":3630336},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":257381},"flush":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1102780},"search":{"threads":6,"queue":0,"active":0,"rejected":0,"largest":6,"completed":12312197},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":28617},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":229333}},"network":{"tcp":{"active_opens":49105,"passive_opens":32875,"curr_estab":450,"in_segs":236219330,"out_segs":222071469,"retrans_segs":1153,"estab_resets":150,"attempt_fails":0,"in_errs":2,"out_rsts":9208}},"fs":{"timestamp":1414493684338,"total":{"total_in_bytes":214643507200,"free_in_bytes":185328312320,"available_in_bytes":185328312320,"disk_reads":308058,"disk_writes":18875331,"disk_io_op":19183389,"disk_read_size_in_bytes":9409968128,"disk_write_size_in_bytes":278838778880,"disk_io_size_in_bytes":288248747008,"disk_queue":"0","disk_service_time":"0"},"data":[{"path":"/vol/data1/elasticsearch/nodes/0","mount":"/vol/data1","dev":"/dev/xvdi","total_in_bytes":53660876800,"free_in_bytes":46313062400,"available_in_bytes":46313062400,"disk_reads":80825,"disk_writes":5783579,"disk_io_op":5864404,"disk_read_size_in_bytes":2508610048,"disk_write_size_in_bytes":99152824832,"disk_io_size_in_bytes":101661434880,"disk_queue":"0","disk_service_time":"0"},{"path":"/vol/data2/elasticsearch/nodes/0","mount":"/vol/data2","dev":"/dev/xvdj","total_in_bytes":53660876800,"free_in_bytes":46340730880,"available_in_bytes":46340730880,"disk_reads":79544,"disk_writes":4169772,"disk_io_op":4249316,"disk_read_size_in_bytes":2378439168,"disk_write_size_in_bytes":63475678208,"disk_io_size_in_bytes":65854117376,"disk_queue":"0","disk_service_time":"0"},{"path":"/vol/data3/elasticsearch/nodes/0","mount":"/vol/data3","dev":"/dev/xvdk","total_in_bytes":53660876800,"free_in_bytes":46333976576,"available_in_bytes":46333976576,"disk_reads":67357,"disk_writes":4372198,"disk_io_op":4439555,"disk_read_size_in_bytes":1932179968,"disk_write_size_in_bytes":56644520448,"disk_io_size_in_bytes":58576700416,"disk_queue":"0","disk_service_time":"0"},{"path":"/vol/data4/elasticsearch/nodes/0","mount":"/vol/data4","dev":"/dev/xvdl","total_in_bytes":53660876800,"free_in_bytes":46340542464,"available_in_bytes":46340542464,"disk_reads":80332,"disk_writes":4549782,"disk_io_op":4630114,"disk_read_size_in_bytes":2590738944,"disk_write_size_in_bytes":59565755392,"disk_io_size_in_bytes":62156494336,"disk_queue":"0","disk_service_time":"0"}]},"transport":{"server_open":408,"rx_count":65756743,"rx_size_in_bytes":11809077539,"tx_count":65756163,"tx_size_in_bytes":21770886999},"http":{"current_open":0,"total_opened":5705},"fielddata_breaker":{"maximum_size_in_bytes":5668208640,"maximum_size":"5.2gb","estimated_size_in_bytes":132573594,"estimated_size":"126.4mb","overhead":1.03,"tripped":0}},"LNlCcpOwSHafvu7OpO4Xaw":{"timestamp":1414493683899,"name":"elasticsearch1","transport_address":"inet[/10.20.147.248:9300]","host":"elasticsearch1","ip":["inet[/10.20.147.248:9300]","NONE"],"attributes":{"aws_availability_zone":"us-west-2a","max_local_storage_nodes":"1"},"indices":{"docs":{"count":65997707,"deleted":12844465},"store":{"size_in_bytes":28928954169,"throttle_time_in_millis":2251},"indexing":{"index_total":10949691,"index_time_in_millis":4699380,"index_current":0,"delete_total":7,"delete_time_in_millis":1,"delete_current":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10108916,"query_time_in_millis":1444802,"query_current":0,"fetch_total":181190,"fetch_time_in_millis":30069,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":25112,"total_time_in_millis":6439447,"total_docs":134080829,"total_size_in_bytes":52054973656},"refresh":{"total":228962,"total_time_in_millis":3090212},"flush":{"total":5105,"total_time_in_millis":398294},"warmer":{"current":0,"total":489673,"total_time_in_millis":57712},"filter_cache":{"memory_size_in_bytes":501758104,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":132260685,"evictions":0,"fields":{"attr_hw_pow_s":{"memory_size_in_bytes":59349988},"attr_allcategories_y":{"memory_size_in_bytes":2612345},"field_updated_l":{"memory_size_in_bytes":36718200},"field_added_l":{"memory_size_in_bytes":33580152}}},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":559,"memory_in_bytes":827691064,"index_writer_memory_in_bytes":3959979,"version_map_memory_in_bytes":15968},"translog":{"operations":46269,"size_in_bytes":0},"suggest":{"total":0,"time_in_millis":0,"current":0}},"os":{"timestamp":1414493684415,"uptime_in_millis":935413,"load_average":[1.57,1.45,1.33],"cpu":{"sys":1,"user":97,"idle":0,"usage":98,"stolen":0},"mem":{"free_in_bytes":326463488,"used_in_bytes":15448887296,"free_percent":29,"used_percent":70,"actual_free_in_bytes":4699906048,"actual_used_in_bytes":11075444736},"swap":{"used_in_bytes":0,"free_in_bytes":0}},"process":{"timestamp":1414493684416,"open_file_descriptors":2232,"cpu":{"percent":198,"sys_in_millis":2591460,"user_in_millis":76801920,"total_in_millis":79393380},"mem":{"resident_in_bytes":10442461184,"share_in_bytes":461733888,"total_virtual_in_bytes":22894739456}},"jvm":{"timestamp":1414493684418,"uptime_in_millis":341843915,"mem":{"heap_used_in_bytes":7473852736,"heap_used_percent":79,"heap_committed_in_bytes":9447014400,"heap_max_in_bytes":9447014400,"non_heap_used_in_bytes":67119856,"non_heap_committed_in_bytes":100728832,"pools":{"young":{"used_in_bytes":5154544,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":10929152,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":7457769040,"max_in_bytes":9289990144,"peak_used_in_bytes":7468410944,"peak_max_in_bytes":9289990144}}},"threads":{"count":50,"peak_count":55},"gc":{"collectors":{"young":{"collection_count":51526,"collection_time_in_millis":3213469},"old":{"collection_count":1071,"collection_time_in_millis":148707}}},"buffer_pools":{"direct":{"count":2512,"used_in_bytes":50531787,"total_capacity_in_bytes":50531787},"mapped":{"count":656,"used_in_bytes":11952062961,"total_capacity_in_bytes":11952062961}}},"thread_pool":{"generic":{"threads":6,"queue":0,"active":1,"rejected":0,"largest":13,"completed":2540252},"index":{"threads":2,"queue":0,"active":0,"rejected":0,"largest":2,"completed":10950050},"snapshot_data":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bench":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":3624563},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":256983},"flush":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1100859},"search":{"threads":6,"queue":0,"active":0,"rejected":0,"largest":6,"completed":12312655},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":4668},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":228964}},"network":{"tcp":{"active_opens":49608,"passive_opens":36078,"curr_estab":457,"in_segs":237600706,"out_segs":226542206,"retrans_segs":22103,"estab_resets":154,"attempt_fails":14,"in_errs":0,"out_rsts":17395}},"fs":{"timestamp":1414493684419,"total":{"total_in_bytes":214643507200,"free_in_bytes":185503555584,"available_in_bytes":185503555584,"disk_reads":1318668,"disk_writes":18966740,"disk_io_op":20285408,"disk_read_size_in_bytes":51807324160,"disk_write_size_in_bytes":281777348096,"disk_io_size_in_bytes":333584672256,"disk_queue":"0","disk_service_time":"0"},"data":[{"path":"/vol/data1/elasticsearch/nodes/0","mount":"/vol/data1","dev":"/dev/xvdi","total_in_bytes":53660876800,"free_in_bytes":46369116160,"available_in_bytes":46369116160,"disk_reads":331053,"disk_writes":4435441,"disk_io_op":4766494,"disk_read_size_in_bytes":13004167680,"disk_write_size_in_bytes":57782944768,"disk_io_size_in_bytes":70787112448,"disk_queue":"0","disk_service_time":"0"},{"path":"/vol/data2/elasticsearch/nodes/0","mount":"/vol/data2","dev":"/dev/xvdj","total_in_bytes":53660876800,"free_in_bytes":46377963520,"available_in_bytes":46377963520,"disk_reads":335992,"disk_writes":4946419,"disk_io_op":5282411,"disk_read_size_in_bytes":13094050304,"disk_write_size_in_bytes":67125739520,"disk_io_size_in_bytes":80219789824,"disk_queue":"0","disk_service_time":"0"},{"path":"/vol/data3/elasticsearch/nodes/0","mount":"/vol/data3","dev":"/dev/xvdk","total_in_bytes":53660876800,"free_in_bytes":46378012672,"available_in_bytes":46378012672,"disk_reads":331405,"disk_writes":4688089,"disk_io_op":5019494,"disk_read_size_in_bytes":12960053760,"disk_write_size_in_bytes":79873139712,"disk_io_size_in_bytes":92833193472,"disk_queue":"0","disk_service_time":"0"},{"path":"/vol/data4/elasticsearch/nodes/0","mount":"/vol/data4","dev":"/dev/xvdl","total_in_bytes":53660876800,"free_in_bytes":46378463232,"available_in_bytes":46378463232,"disk_reads":320218,"disk_writes":4896791,"disk_io_op":5217009,"disk_read_size_in_bytes":12749052416,"disk_write_size_in_bytes":76995524096,"disk_io_size_in_bytes":89744576512,"disk_queue":"0","disk_service_time":"0"}]},"transport":{"server_open":408,"rx_count":50424171,"rx_size_in_bytes":14499304531,"tx_count":50420307,"tx_size_in_bytes":7747488441},"http":{"current_open":5,"total_opened":5818},"fielddata_breaker":{"maximum_size_in_bytes":5668208640,"maximum_size":"5.2gb","estimated_size_in_bytes":132260685,"estimated_size":"126.1mb","overhead":1.03,"tripped":0}},"vs3KTjwtRTycfP-GwaAniw":{"timestamp":1414493683901,"name":"elasticsearch3","transport_address":"inet[/10.20.151.176:9300]","host":"elasticsearch3","ip":["inet[/10.20.151.176:9300]","NONE"],"attributes":{"aws_availability_zone":"us-west-2c","max_local_storage_nodes":"1"},"indices":{"docs":{"count":65997369,"deleted":12822537},"store":{"size_in_bytes":28927136631,"throttle_time_in_millis":964},"indexing":{"index_total":10983993,"index_time_in_millis":3967137,"index_current":0,"delete_total":7,"delete_time_in_millis":1,"delete_current":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10114812,"query_time_in_millis":1124799,"query_current":0,"fetch_total":180811,"fetch_time_in_millis":23622,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":25083,"total_time_in_millis":5999499,"total_docs":136122624,"total_size_in_bytes":52806637514},"refresh":{"total":229650,"total_time_in_millis":2875688},"flush":{"total":5105,"total_time_in_millis":276084},"warmer":{"current":0,"total":491042,"total_time_in_millis":35073},"filter_cache":{"memory_size_in_bytes":493619848,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":133164974,"evictions":0,"fields":{"field_updated_l":{"memory_size_in_bytes":36823384},"attr_allcategories_y":{"memory_size_in_bytes":2603897},"attr_hw_pow_s":{"memory_size_in_bytes":59657921},"field_added_l":{"memory_size_in_bytes":34079772}}},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":537,"memory_in_bytes":826375904,"index_writer_memory_in_bytes":5750365,"version_map_memory_in_bytes":91696},"translog":{"operations":61339,"size_in_bytes":0},"suggest":{"total":0,"time_in_millis":0,"current":0}},"os":{"timestamp":1414493684231,"uptime_in_millis":933884,"load_average":[1.32,1.39,1.33],"cpu":{"sys":1,"user":54,"idle":43,"usage":55,"stolen":0},"mem":{"free_in_bytes":273350656,"used_in_bytes":15502000128,"free_percent":30,"used_percent":69,"actual_free_in_bytes":4752486400,"actual_used_in_bytes":11022864384},"swap":{"used_in_bytes":0,"free_in_bytes":0}},"process":{"timestamp":1414493684232,"open_file_descriptors":2200,"cpu":{"percent":112,"sys_in_millis":2369080,"user_in_millis":57050180,"total_in_millis":59419260},"mem":{"resident_in_bytes":10401804288,"share_in_bytes":448081920,"total_virtual_in_bytes":22885699584}},"jvm":{"timestamp":1414493684233,"uptime_in_millis":342684306,"mem":{"heap_used_in_bytes":7353266456,"heap_used_percent":77,"heap_committed_in_bytes":9447014400,"heap_max_in_bytes":9447014400,"non_heap_used_in_bytes":66080168,"non_heap_committed_in_bytes":99549184,"pools":{"young":{"used_in_bytes":110379384,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":7225454496,"max_in_bytes":9289990144,"peak_used_in_bytes":7230027928,"peak_max_in_bytes":9289990144}}},"threads":{"count":52,"peak_count":54},"gc":{"collectors":{"young":{"collection_count":52584,"collection_time_in_millis":3226872},"old":{"collection_count":553,"collection_time_in_millis":64675}}},"buffer_pools":{"direct":{"count":2488,"used_in_bytes":47149640,"total_capacity_in_bytes":47149640},"mapped":{"count":650,"used_in_bytes":11943246614,"total_capacity_in_bytes":11943246614}}},"thread_pool":{"generic":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":11,"completed":2535807},"index":{"threads":2,"queue":0,"active":0,"rejected":0,"largest":2,"completed":10984353},"snapshot_data":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bench":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":3633767},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":257634},"flush":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1103815},"search":{"threads":6,"queue":0,"active":0,"rejected":0,"largest":6,"completed":12318102},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":8624},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":229653}},"network":{"tcp":{"active_opens":49307,"passive_opens":34505,"curr_estab":450,"in_segs":226913235,"out_segs":197536260,"retrans_segs":1167,"estab_resets":141,"attempt_fails":0,"in_errs":3,"out_rsts":6529}},"fs":{"timestamp":1414493684234,"total":{"total_in_bytes":214643507200,"free_in_bytes":185495404544,"available_in_bytes":185495404544,"disk_reads":319719,"disk_writes":19090578,"disk_io_op":19410297,"disk_read_size_in_bytes":8159500288,"disk_write_size_in_bytes":279031630336,"disk_io_size_in_bytes":287191130624,"disk_queue":"0","disk_service_time":"14.4"},"data":[{"path":"/vol/data1/elasticsearch/nodes/0","mount":"/vol/data1","dev":"/dev/xvdi","total_in_bytes":53660876800,"free_in_bytes":46372122624,"available_in_bytes":46372122624,"disk_reads":93769,"disk_writes":4661427,"disk_io_op":4755196,"disk_read_size_in_bytes":2333735424,"disk_write_size_in_bytes":58518721024,"disk_io_size_in_bytes":60852456448,"disk_queue":"0","disk_service_time":"3.6"},{"path":"/vol/data2/elasticsearch/nodes/0","mount":"/vol/data2","dev":"/dev/xvdj","total_in_bytes":53660876800,"free_in_bytes":46372929536,"available_in_bytes":46372929536,"disk_reads":68626,"disk_writes":4630843,"disk_io_op":4699469,"disk_read_size_in_bytes":1731922432,"disk_write_size_in_bytes":46415779328,"disk_io_size_in_bytes":48147701760,"disk_queue":"0","disk_service_time":"3.6"},{"path":"/vol/data3/elasticsearch/nodes/0","mount":"/vol/data3","dev":"/dev/xvdk","total_in_bytes":53660876800,"free_in_bytes":46375186432,"available_in_bytes":46375186432,"disk_reads":91963,"disk_writes":6202492,"disk_io_op":6294455,"disk_read_size_in_bytes":2385357312,"disk_write_size_in_bytes":115548634112,"disk_io_size_in_bytes":117933991424,"disk_queue":"0","disk_service_time":"3.6"},{"path":"/vol/data4/elasticsearch/nodes/0","mount":"/vol/data4","dev":"/dev/xvdl","total_in_bytes":53660876800,"free_in_bytes":46375165952,"available_in_bytes":46375165952,"disk_reads":65361,"disk_writes":3595816,"disk_io_op":3661177,"disk_read_size_in_bytes":1708485120,"disk_write_size_in_bytes":58548495872,"disk_io_size_in_bytes":60256980992,"disk_queue":"0","disk_service_time":"3.6"}]},"transport":{"server_open":408,"rx_count":50495990,"rx_size_in_bytes":14286270224,"tx_count":50495461,"tx_size_in_bytes":7761581980},"http":{"current_open":0,"total_opened":7044},"fielddata_breaker":{"maximum_size_in_bytes":5668208640,"maximum_size":"5.2gb","estimated_size_in_bytes":133164974,"estimated_size":"126.9mb","overhead":1.03,"tripped":0}}}}
@johnc10uk
Copy link
Author

::: [elasticsearch2][kxyKdBB5Ri6AIx-gexM-NQ][elasticsearch2][inet[/x.x.x.x:9300]]

0.7% (3.3ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch2][index][T#1]'
 10/10 snapshots sharing following 10 elements
   sun.misc.Unsafe.park(Native Method)
   java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735)
   java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644)
   java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137)
   org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
   java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

0.5% (2.5ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch2][index][T#2]'
 9/10 snapshots sharing following 10 elements
   sun.misc.Unsafe.park(Native Method)
   java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735)
   java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644)
   java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137)
   org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
   java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)
 unique snapshot
   sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
   sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
   sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.registerTask(AbstractNioSelector.java:115)
   org.elasticsearch.common.netty.channel.socket.nio.NioWorker.scheduleWriteIfNecessary(NioWorker.java:106)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.writeFromUserCode(AbstractNioWorker.java:132)
   org.elasticsearch.common.netty.channel.socket.nio.NioClientSocketPipelineSink.eventSunk(NioClientSocketPipelineSink.java:84)
   org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:574)
   org.elasticsearch.common.netty.channel.Channels.write(Channels.java:704)
   org.elasticsearch.common.netty.channel.Channels.write(Channels.java:671)
   org.elasticsearch.common.netty.channel.AbstractChannel.write(AbstractChannel.java:248)
   org.elasticsearch.transport.netty.NettyTransport.sendRequest(NettyTransport.java:607)
   org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:198)
   org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnReplica(TransportShardReplicationOperationAction.java:693)
   org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performReplicas(TransportShardReplicationOperationAction.java:665)
   org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:536)
   org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:434)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

0.2% (825.6micros out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch2][transport_server_worker][T#2]{New I/O worker #7}'
 10/10 snapshots sharing following 15 elements
   sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
   sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
   org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
   org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
   org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

::: [elasticsearch1][LNlCcpOwSHafvu7OpO4Xaw][elasticsearch1][inet[/x.x.x.x:9300]

0.6% (2.8ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch1][index][T#2]'
 10/10 snapshots sharing following 10 elements
   sun.misc.Unsafe.park(Native Method)
   java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735)
   java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644)
   java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137)
   org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
   java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

0.4% (2.2ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch1][index][T#1]'
 10/10 snapshots sharing following 10 elements
   sun.misc.Unsafe.park(Native Method)
   java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735)
   java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644)
   java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137)
   org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
   java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

0.3% (1.4ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch1][transport_server_worker][T#2]{New I/O worker #7}'
 10/10 snapshots sharing following 15 elements
   sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
   sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
   org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
   org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
   org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

::: [elasticsearch3][vs3KTjwtRTycfP-GwaAniw][elasticsearch3][inet[/x.x.x.x:9300]]

0.6% (2.9ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch3][index][T#2]'
 10/10 snapshots sharing following 10 elements
   sun.misc.Unsafe.park(Native Method)
   java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735)
   java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644)
   java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137)
   org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
   java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

0.3% (1.7ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch3][index][T#1]'
 10/10 snapshots sharing following 10 elements
   sun.misc.Unsafe.park(Native Method)
   java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:735)
   java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:644)
   java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1137)
   org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:162)
   java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

0.3% (1.2ms out of 500ms) cpu usage by thread 'elasticsearch[elasticsearch3][transport_server_worker][T#4]{New I/O worker #9}'
 10/10 snapshots sharing following 15 elements
   sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
   sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
   sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   org.elasticsearch.common.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
   org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
   org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
   org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
   org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
   java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   java.lang.Thread.run(Thread.java:745)

@johnc10uk
Copy link
Author

Node Name: elasticsearch1 elasticsearch2 elasticsearch3
IP Address: 10.20.147.248:9300 10.20.149.92:9300 10.20.151.176:9300
Node ID: pL_NGG7wTyuHPkPEblQ0Mw kxyKdBB5Ri6AIx-gexM-NQ cE6uMYMoQ3iCKOLnN5eBVQ
ES Uptime: 0.87 days 4.90 days 0.87 days
File System
Store Size: 27.2GB 27.2GB 27.2GB

Documents: 66,517,273 66,517,267 66,517,241

Documents Deleted: 19.6% 19.7% 19.6%
Merge Size: 11.4GB 62.0GB 11.2GB
Merge Time: 00:22:31 02:11:42 00:21:14
Merge Rate: 9 MB/s 8.4 MB/s 9.5 MB/s
File Descriptors: 2,341 2,229 2,221
Index Activity
Indexing - Index: 0.42ms 0.38ms 0.35ms
Indexing - Delete: 0.5ms 0.11ms 0.5ms
Search - Query: 0.14ms 0.11ms 0.11ms
Search - Fetch: 0.18ms 0.14ms 0.13ms
Get - Total: 0ms 0ms 0ms
Get - Exists: 0ms 0ms 0ms
Get - Missing: 0ms 0ms 0ms
Refresh: 13.56ms 13.19ms 12.96ms
Flush: 82.29ms 55.54ms 61.76ms
Cache Activity
Field Size: 118.1MB 108.3MB 93.3MB
Field Evictions: 0 0 0
Filter Cache Size: 322.0MB 320.8MB 321.4MB
Filter Evictions: 0 per query 0 per query 0 per query
ID Cache Size:
% ID Cache: 0% 0% 0%
Memory
Total Memory: 15 gb 15 gb 15 gb
Heap Size: 8.8 gb 8.8 gb 8.8 gb
Heap % of RAM: 59.9% 59.9% 59.9%
% Heap Used: 40.7% 63.5% 45.1%
GC MarkSweep Frequency: 0 s 0 s 0 s
GC MarkSweep Duration: 0ms 0ms 0ms
GC ParNew Frequency: 0 s 0 s 0 s
GC ParNew Duration: 0ms 0ms 0ms
G1 GC Young Generation Freq: 0 s 0 s 0 s
G1 GC Young Generation Duration: 0ms 0ms 0ms
G1 GC Old Generation Freq: 0 s 0 s 0 s
G1 GC Old Generation Duration: 0ms 0ms 0ms
Swap Space: 0.0000 mb 0.0000 mb 0.0000 mb
Network
HTTP Connection Rate: 0 /second 0 /second 0 /second

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment