node-stats.json

{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.7gb","size_in_bytes":19015420480,"docs":{"num_docs":5824080},"cache":{"field_evictions":0,"field_size":"1.8gb","field_size_in_bytes":1941834588,"filter_count":452761,"filter_evictions":0,"filter_mem_evictions":565,"filter_size":"978.4mb","filter_size_in_bytes":1025931880},"merges":{"current":0,"total":383,"total_time":"32.8s","total_time_in_millis":32804}},"os":{"timestamp":1303069801270,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303069801270},"jvm":{"timestamp":1303069801270,"uptime":"1 hour, 58 minutes, 31 seconds and 249 milliseconds","uptime_in_millis":7111249,"mem":{"heap_used":"4.3gb","heap_used_in_bytes":4662957496,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"40.5mb","non_heap_used_in_bytes":42482096,"non_heap_committed":"41.3mb","non_heap_committed_in_bytes":43384832},"threads":{"count":63,"peak_count":98},"gc":{"collection_count":420,"collection_time":"8 seconds and 88 milliseconds","collection_time_in_millis":8088,"collectors":{"ParNew":{"collection_count":418,"collection_time":"8 seconds and 88 milliseconds","collection_time_in_millis":8088},"ConcurrentMarkSweep":{"collection_count":2,"collection_time":"0 milliseconds","collection_time_in_millis":0}}}},"network":{},"transport":{"rx_count":37835,"rx_size":"21.4mb","rx_size_in_bytes":22501194,"tx_count":39966,"tx_size":"241.7mb","tx_size_in_bytes":253464819}}}} Sun Apr 17 21:51:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.7gb","size_in_bytes":19015941108,"docs":{"num_docs":5824366},"cache":{"field_evictions":0,"field_size":"1.8gb","field_size_in_bytes":1941853204,"filter_count":458297,"filter_evictions":0,"filter_mem_evictions":565,"filter_size":"986.7mb","filter_size_in_bytes":1034672544},"merges":{"current":0,"total":386,"total_time":"32.8s","total_time_in_millis":32842}},"os":{"timestamp":1303069861361,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303069861361},"jvm":{"timestamp":1303069861361,"uptime":"1 hour, 59 minutes, 31 seconds and 340 milliseconds","uptime_in_millis":7171340,"mem":{"heap_used":"4.3gb","heap_used_in_bytes":4626235808,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"40.5mb","non_heap_used_in_bytes":42483984,"non_heap_committed":"41.3mb","non_heap_committed_in_bytes":43384832},"threads":{"count":59,"peak_count":98},"gc":{"collection_count":423,"collection_time":"8 seconds and 154 milliseconds","collection_time_in_millis":8154,"collectors":{"ParNew":{"collection_count":421,"collection_time":"8 seconds and 154 milliseconds","collection_time_in_millis":8154},"ConcurrentMarkSweep":{"collection_count":2,"collection_time":"0 milliseconds","collection_time_in_millis":0}}}},"network":{},"transport":{"rx_count":38218,"rx_size":"21.7mb","rx_size_in_bytes":22766512,"tx_count":40349,"tx_size":"242.6mb","tx_size_in_bytes":254401593}}}} Sun Apr 17 21:52:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.6gb","size_in_bytes":18972759285,"docs":{"num_docs":5824676},"cache":{"field_evictions":0,"field_size":"1.8gb","field_size_in_bytes":1938384118,"filter_count":462925,"filter_evictions":0,"filter_mem_evictions":565,"filter_size":"993.2mb","filter_size_in_bytes":1041448616},"merges":{"current":0,"total":389,"total_time":"33.1s","total_time_in_millis":33189}},"os":{"timestamp":1303069921458,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303069921458},"jvm":{"timestamp":1303069921458,"uptime":"2 hours, 31 seconds and 437 milliseconds","uptime_in_millis":7231437,"mem":{"heap_used":"4.3gb","heap_used_in_bytes":4716728720,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"40.5mb","non_heap_used_in_bytes":42507792,"non_heap_committed":"41.3mb","non_heap_committed_in_bytes":43384832},"threads":{"count":79,"peak_count":98},"gc":{"collection_count":449,"collection_time":"8 seconds and 544 milliseconds","collection_time_in_millis":8544,"collectors":{"ParNew":{"collection_count":447,"collection_time":"8 seconds and 544 milliseconds","collection_time_in_millis":8544},"ConcurrentMarkSweep":{"collection_count":2,"collection_time":"0 milliseconds","collection_time_in_millis":0}}}},"network":{},"transport":{"rx_count":38597,"rx_size":"21.9mb","rx_size_in_bytes":23015942,"tx_count":40752,"tx_size":"245mb","tx_size_in_bytes":256955061}}}} Sun Apr 17 21:53:01 CEST 2011
...

regular optimize at 07:29 which does not result in a crash but node stats didn't report a response for about 2 minutes

{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.8gb","size_in_bytes":19181174535,"docs":{"num_docs":5484018},"cache":{"field_evictions":193,"field_size":"1.7gb","field_size_in_bytes":1900422140,"filter_count":1485612,"filter_evictions":0,"filter_mem_evictions":20015,"filter_size":"4.5gb","filter_size_in_bytes":4856386280},"merges":{"current":0,"total":2284,"total_time":"2.3m","total_time_in_millis":139656}},"os":{"timestamp":1303104697780,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303104697780},"jvm":{"timestamp":1303104697780,"uptime":"11 hours, 40 minutes, 7 seconds and 759 milliseconds","uptime_in_millis":42007759,"mem":{"heap_used":"7gb","heap_used_in_bytes":7520779712,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.1mb","non_heap_used_in_bytes":43103376,"non_heap_committed":"64.5mb","non_heap_committed_in_bytes":67670016},"threads":{"count":81,"peak_count":116},"gc":{"collection_count":4348,"collection_time":"7 minutes, 13 seconds and 251 milliseconds","collection_time_in_millis":433251,"collectors":{"ParNew":{"collection_count":2082,"collection_time":"55 seconds and 845 milliseconds","collection_time_in_millis":55845},"ConcurrentMarkSweep":{"collection_count":2266,"collection_time":"6 minutes, 17 seconds and 406 milliseconds","collection_time_in_millis":377406}}}},"network":{},"transport":{"rx_count":222358,"rx_size":"132.2mb","rx_size_in_bytes":138634549,"tx_count":234864,"tx_size":"1.3gb","tx_size_in_bytes":1463279576}}}} Mon Apr 18 07:32:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.8gb","size_in_bytes":19181445025,"docs":{"num_docs":5484149},"cache":{"field_evictions":235,"field_size":"1.7gb","field_size_in_bytes":1900434372,"filter_count":1487694,"filter_evictions":0,"filter_mem_evictions":20070,"filter_size":"4.5gb","filter_size_in_bytes":4858311960},"merges":{"current":0,"total":2285,"total_time":"2.3m","total_time_in_millis":139846}},"os":{"timestamp":1303104755941,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303104755941},"jvm":{"timestamp":1303104755941,"uptime":"11 hours, 41 minutes, 5 seconds and 920 milliseconds","uptime_in_millis":42065920,"mem":{"heap_used":"7gb","heap_used_in_bytes":7529136544,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.1mb","non_heap_used_in_bytes":43103376,"non_heap_committed":"64.5mb","non_heap_committed_in_bytes":67670016},"threads":{"count":82,"peak_count":116},"gc":{"collection_count":4378,"collection_time":"7 minutes, 21 seconds and 880 milliseconds","collection_time_in_millis":441880,"collectors":{"ParNew":{"collection_count":2082,"collection_time":"55 seconds and 845 milliseconds","collection_time_in_millis":55845},"ConcurrentMarkSweep":{"collection_count":2296,"collection_time":"6 minutes, 26 seconds and 35 milliseconds","collection_time_in_millis":386035}}}},"network":{},"transport":{"rx_count":222532,"rx_size":"132.2mb","rx_size_in_bytes":138722462,"tx_count":235039,"tx_size":"1.3gb","tx_size_in_bytes":1463835515}}}} Mon Apr 18 07:33:01 CEST 2011
<no response> Mon Apr 18 07:34:02 CEST 2011
<no response> Mon Apr 18 07:35:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.8gb","size_in_bytes":19181774913,"docs":{"num_docs":5484350},"cache":{"field_evictions":301,"field_size":"1.7gb","field_size_in_bytes":1900458478,"filter_count":190028,"filter_evictions":0,"filter_mem_evictions":20167,"filter_size":"36.6mb","filter_size_in_bytes":38427088},"merges":{"current":0,"total":2290,"total_time":"2.6m","total_time_in_millis":158898}},"os":{"timestamp":1303104920719,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303104920719},"jvm":{"timestamp":1303104920808,"uptime":"11 hours, 43 minutes, 50 seconds and 787 milliseconds","uptime_in_millis":42230787,"mem":{"heap_used":"693.7mb","heap_used_in_bytes":727434472,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.1mb","non_heap_used_in_bytes":43110760,"non_heap_committed":"64.5mb","non_heap_committed_in_bytes":67670016},"threads":{"count":82,"peak_count":116},"gc":{"collection_count":4476,"collection_time":"7 minutes, 48 seconds and 866 milliseconds","collection_time_in_millis":468866,"collectors":{"ParNew":{"collection_count":2082,"collection_time":"55 seconds and 845 milliseconds","collection_time_in_millis":55845},"ConcurrentMarkSweep":{"collection_count":2394,"collection_time":"6 minutes, 53 seconds and 21 milliseconds","collection_time_in_millis":413021}}}},"network":{},"transport":{"rx_count":222854,"rx_size":"132.4mb","rx_size_in_bytes":138873973,"tx_count":235432,"tx_size":"1.3gb","tx_size_in_bytes":1470200181}}}} Mon Apr 18 07:36:01 CEST 2011
...
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.7gb","size_in_bytes":19042070964,"docs":{"num_docs":5489328},"cache":{"field_evictions":771,"field_size":"2.2gb","field_size_in_bytes":2465938510,"filter_count":286748,"filter_evictions":0,"filter_mem_evictions":20364,"filter_size":"260.9mb","filter_size_in_bytes":273601408},"merges":{"current":1,"total":2368,"total_time":"24.4m","total_time_in_millis":1469815}},"os":{"timestamp":1303107541479,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303107541479},"jvm":{"timestamp":1303107541479,"uptime":"12 hours, 27 minutes, 31 seconds and 458 milliseconds","uptime_in_millis":44851458,"mem":{"heap_used":"3.5gb","heap_used_in_bytes":3817024296,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.2mb","non_heap_used_in_bytes":43245600,"non_heap_committed":"64.6mb","non_heap_committed_in_bytes":67801088},"threads":{"count":65,"peak_count":116},"gc":{"collection_count":4828,"collection_time":"7 minutes, 53 seconds and 645 milliseconds","collection_time_in_millis":473645,"collectors":{"ParNew":{"collection_count":2433,"collection_time":"1 minute and 618 milliseconds","collection_time_in_millis":60618},"ConcurrentMarkSweep":{"collection_count":2395,"collection_time":"6 minutes, 53 seconds and 27 milliseconds","collection_time_in_millis":413027}}}},"network":{},"transport":{"rx_count":231356,"rx_size":"136.8mb","rx_size_in_bytes":143505011,"tx_count":244302,"tx_size":"1.4gb","tx_size_in_bytes":1525264551}}}} Mon Apr 18 08:20:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"17.9gb","size_in_bytes":19236581812,"docs":{"num_docs":5489328},"cache":{"field_evictions":771,"field_size":"2.2gb","field_size_in_bytes":2465938510,"filter_count":286868,"filter_evictions":0,"filter_mem_evictions":20364,"filter_size":"262.5mb","filter_size_in_bytes":275286360},"merges":{"current":1,"total":2368,"total_time":"24.4m","total_time_in_millis":1469815}},"os":{"timestamp":1303107601708,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303107601708},"jvm":{"timestamp":1303107601708,"uptime":"12 hours, 28 minutes, 31 seconds and 687 milliseconds","uptime_in_millis":44911687,"mem":{"heap_used":"3.5gb","heap_used_in_bytes":3838905136,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.2mb","non_heap_used_in_bytes":43245600,"non_heap_committed":"64.6mb","non_heap_committed_in_bytes":67801088},"threads":{"count":64,"peak_count":116},"gc":{"collection_count":4839,"collection_time":"7 minutes, 53 seconds and 773 milliseconds","collection_time_in_millis":473773,"collectors":{"ParNew":{"collection_count":2444,"collection_time":"1 minute and 746 milliseconds","collection_time_in_millis":60746},"ConcurrentMarkSweep":{"collection_count":2395,"collection_time":"6 minutes, 53 seconds and 27 milliseconds","collection_time_in_millis":413027}}}},"network":{},"transport":{"rx_count":231438,"rx_size":"136.8mb","rx_size_in_bytes":143521487,"tx_count":244384,"tx_size":"1.4gb","tx_size_in_bytes":1525796324}}}} Mon Apr 18 08:21:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.6gb","size_in_bytes":7146412376,"docs":{"num_docs":5490997},"cache":{"field_evictions":771,"field_size":"2.5gb","field_size_in_bytes":2710911404,"filter_count":288909,"filter_evictions":0,"filter_mem_evictions":20364,"filter_size":"337.3mb","filter_size_in_bytes":353746144},"merges":{"current":0,"total":2370,"total_time":"30.6m","total_time_in_millis":1837890}},"os":{"timestamp":1303107661868,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303107661868},"jvm":{"timestamp":1303107661868,"uptime":"12 hours, 29 minutes, 31 seconds and 847 milliseconds","uptime_in_millis":44971847,"mem":{"heap_used":"3.9gb","heap_used_in_bytes":4231080144,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.2mb","non_heap_used_in_bytes":43246656,"non_heap_committed":"64.6mb","non_heap_committed_in_bytes":67801088},"threads":{"count":72,"peak_count":116},"gc":{"collection_count":4865,"collection_time":"7 minutes, 54 seconds and 297 milliseconds","collection_time_in_millis":474297,"collectors":{"ParNew":{"collection_count":2470,"collection_time":"1 minute, 1 second and 270 milliseconds","collection_time_in_millis":61270},"ConcurrentMarkSweep":{"collection_count":2395,"collection_time":"6 minutes, 53 seconds and 27 milliseconds","collection_time_in_millis":413027}}}},"network":{},"transport":{"rx_count":232872,"rx_size":"137.8mb","rx_size_in_bytes":144558751,"tx_count":245819,"tx_size":"1.4gb","tx_size_in_bytes":1527421195}}}} Mon Apr 18 08:22:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.6gb","size_in_bytes":7148808952,"docs":{"num_docs":5492314},"cache":{"field_evictions":771,"field_size":"2.5gb","field_size_in_bytes":2711001660,"filter_count":293055,"filter_evictions":0,"filter_mem_evictions":20364,"filter_size":"384.2mb","filter_size_in_bytes":402928432},"merges":{"current":0,"total":2371,"total_time":"30.6m","total_time_in_millis":1837897}},"os":{"timestamp":1303107722148,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303107722148},"jvm":{"timestamp":1303107722148,"uptime":"12 hours, 30 minutes, 32 seconds and 127 milliseconds","uptime_in_millis":45032127,"mem":{"heap_used":"3.9gb","heap_used_in_bytes":4274619872,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.2mb","non_heap_used_in_bytes":43221008,"non_heap_committed":"64.6mb","non_heap_committed_in_bytes":67801088},"threads":{"count":83,"peak_count":116},"gc":{"collection_count":4877,"collection_time":"7 minutes, 54 seconds and 505 milliseconds","collection_time_in_millis":474505,"collectors":{"ParNew":{"collection_count":2482,"collection_time":"1 minute, 1 second and 478 milliseconds","collection_time_in_millis":61478},"ConcurrentMarkSweep":{"collection_count":2395,"collection_time":"6 minutes, 53 seconds and 27 milliseconds","collection_time_in_millis":413027}}}},"network":{},"transport":{"rx_count":234081,"rx_size":"138.7mb","rx_size_in_bytes":145500903,"tx_count":247029,"tx_size":"1.4gb","tx_size_in_bytes":1528645348}}}} Mon Apr 18 08:23:01 CEST 2011

...
  
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.9gb","size_in_bytes":7428890361,"docs":{"num_docs":5671684},"cache":{"field_evictions":1863,"field_size":"817mb","field_size_in_bytes":856785634,"filter_count":149383,"filter_evictions":0,"filter_mem_evictions":38567,"filter_size":"2gb","filter_size_in_bytes":2242323880},"merges":{"current":0,"total":4412,"total_time":"33.3m","total_time_in_millis":1999732}},"os":{"timestamp":1303146541648,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303146541648},"jvm":{"timestamp":1303146541648,"uptime":"23 hours, 17 minutes, 31 seconds and 627 milliseconds","uptime_in_millis":83851627,"mem":{"heap_used":"3.6gb","heap_used_in_bytes":3945444552,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.5mb","non_heap_used_in_bytes":43538008,"non_heap_committed":"64.8mb","non_heap_committed_in_bytes":67997696},"threads":{"count":64,"peak_count":116},"gc":{"collection_count":8924,"collection_time":"15 minutes, 13 seconds and 709 milliseconds","collection_time_in_millis":913709,"collectors":{"ParNew":{"collection_count":3936,"collection_time":"1 minute, 41 seconds and 567 milliseconds","collection_time_in_millis":101567},"ConcurrentMarkSweep":{"collection_count":4988,"collection_time":"13 minutes, 32 seconds and 142 milliseconds","collection_time_in_millis":812142}}}},"network":{},"transport":{"rx_count":470325,"rx_size":"289.5mb","rx_size_in_bytes":303631753,"tx_count":494223,"tx_size":"2.6gb","tx_size_in_bytes":2885099359}}}} Mon Apr 18 19:10:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.9gb","size_in_bytes":7429338190,"docs":{"num_docs":5671956},"cache":{"field_evictions":1863,"field_size":"817.1mb","field_size_in_bytes":856822218,"filter_count":152064,"filter_evictions":0,"filter_mem_evictions":38575,"filter_size":"2gb","filter_size_in_bytes":2250368792},"merges":{"current":0,"total":4415,"total_time":"33.3m","total_time_in_millis":1999751}},"os":{"timestamp":1303146601731,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303146601731},"jvm":{"timestamp":1303146601731,"uptime":"23 hours, 18 minutes, 31 seconds and 710 milliseconds","uptime_in_millis":83911710,"mem":{"heap_used":"3.7gb","heap_used_in_bytes":3979759944,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.5mb","non_heap_used_in_bytes":43538008,"non_heap_committed":"64.8mb","non_heap_committed_in_bytes":67997696},"threads":{"count":65,"peak_count":116},"gc":{"collection_count":8925,"collection_time":"15 minutes, 13 seconds and 728 milliseconds","collection_time_in_millis":913728,"collectors":{"ParNew":{"collection_count":3937,"collection_time":"1 minute, 41 seconds and 586 milliseconds","collection_time_in_millis":101586},"ConcurrentMarkSweep":{"collection_count":4988,"collection_time":"13 minutes, 32 seconds and 142 milliseconds","collection_time_in_millis":812142}}}},"network":{},"transport":{"rx_count":470658,"rx_size":"289.7mb","rx_size_in_bytes":303819941,"tx_count":494579,"tx_size":"2.6gb","tx_size_in_bytes":2887610578}}}} Mon Apr 18 19:11:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.9gb","size_in_bytes":7429825447,"docs":{"num_docs":5672187},"cache":{"field_evictions":1863,"field_size":"817.1mb","field_size_in_bytes":856847078,"filter_count":155285,"filter_evictions":0,"filter_mem_evictions":38575,"filter_size":"2.1gb","filter_size_in_bytes":2259397904},"merges":{"current":0,"total":4417,"total_time":"33.3m","total_time_in_millis":1999762}},"os":{"timestamp":1303146661800,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303146661800},"jvm":{"timestamp":1303146661800,"uptime":"23 hours, 19 minutes, 31 seconds and 779 milliseconds","uptime_in_millis":83971779,"mem":{"heap_used":"3.7gb","heap_used_in_bytes":4022068136,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.5mb","non_heap_used_in_bytes":43538008,"non_heap_committed":"64.8mb","non_heap_committed_in_bytes":67997696},"threads":{"count":66,"peak_count":116},"gc":{"collection_count":8926,"collection_time":"15 minutes, 13 seconds and 759 milliseconds","collection_time_in_millis":913759,"collectors":{"ParNew":{"collection_count":3938,"collection_time":"1 minute, 41 seconds and 617 milliseconds","collection_time_in_millis":101617},"ConcurrentMarkSweep":{"collection_count":4988,"collection_time":"13 minutes, 32 seconds and 142 milliseconds","collection_time_in_millis":812142}}}},"network":{},"transport":{"rx_count":470949,"rx_size":"289.9mb","rx_size_in_bytes":304020790,"tx_count":494871,"tx_size":"2.6gb","tx_size_in_bytes":2888478014}}}} Mon Apr 18 19:12:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.9gb","size_in_bytes":7430490355,"docs":{"num_docs":5672705},"cache":{"field_evictions":1863,"field_size":"817.1mb","field_size_in_bytes":856808024,"filter_count":161229,"filter_evictions":0,"filter_mem_evictions":38579,"filter_size":"2.1gb","filter_size_in_bytes":2270461232},"merges":{"current":0,"total":4421,"total_time":"33.3m","total_time_in_millis":2000064}},"os":{"timestamp":1303146721887,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303146721887},"jvm":{"timestamp":1303146721887,"uptime":"23 hours, 20 minutes, 31 seconds and 866 milliseconds","uptime_in_millis":84031866,"mem":{"heap_used":"3.7gb","heap_used_in_bytes":4077677600,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.5mb","non_heap_used_in_bytes":43538008,"non_heap_committed":"64.8mb","non_heap_committed_in_bytes":67997696},"threads":{"count":65,"peak_count":116},"gc":{"collection_count":8928,"collection_time":"15 minutes, 13 seconds and 806 milliseconds","collection_time_in_millis":913806,"collectors":{"ParNew":{"collection_count":3940,"collection_time":"1 minute, 41 seconds and 664 milliseconds","collection_time_in_millis":101664},"ConcurrentMarkSweep":{"collection_count":4988,"collection_time":"13 minutes, 32 seconds and 142 milliseconds","collection_time_in_millis":812142}}}},"network":{},"transport":{"rx_count":471446,"rx_size":"290.2mb","rx_size_in_bytes":304350559,"tx_count":495393,"tx_size":"2.6gb","tx_size_in_bytes":2891098599}}}} Mon Apr 18 19:13:01 CEST 2011
{"cluster_name":"jetwickcluster","nodes":{"R57UiAZtRl26lo1pKM-Z-Q":{"name":"Diamanda Nero","indices":{"size":"6.9gb","size_in_bytes":7430912926,"docs":{"num_docs":5672937},"cache":{"field_evictions":1863,"field_size":"817.2mb","field_size_in_bytes":856939604,"filter_count":163630,"filter_evictions":0,"filter_mem_evictions":38584,"filter_size":"2.1gb","filter_size_in_bytes":2278511328},"merges":{"current":0,"total":4425,"total_time":"33.3m","total_time_in_millis":2000088}},"os":{"timestamp":1303146781982,"uptime":"-1 seconds","uptime_in_millis":-1000,"load_average":[]},"process":{"timestamp":1303146781982},"jvm":{"timestamp":1303146781982,"uptime":"23 hours, 21 minutes, 31 seconds and 961 milliseconds","uptime_in_millis":84091961,"mem":{"heap_used":"3.7gb","heap_used_in_bytes":4066586312,"heap_committed":"7gb","heap_committed_in_bytes":7532314624,"non_heap_used":"41.5mb","non_heap_used_in_bytes":43540344,"non_heap_committed":"64.8mb","non_heap_committed_in_bytes":67997696},"threads":{"count":64,"peak_count":116},"gc":{"collection_count":8932,"collection_time":"15 minutes, 13 seconds and 898 milliseconds","collection_time_in_millis":913898,"collectors":{"ParNew":{"collection_count":3944,"collection_time":"1 minute, 41 seconds and 756 milliseconds","collection_time_in_millis":101756},"ConcurrentMarkSweep":{"collection_count":4988,"collection_time":"13 minutes, 32 seconds and 142 milliseconds","collection_time_in_millis":812142}}}},"network":{},"transport":{"rx_count":471778,"rx_size":"290.4mb","rx_size_in_bytes":304573195,"tx_count":495749,"tx_size":"2.6gb","tx_size_in_bytes":2893594198}}}} Mon Apr 18 19:16:01 CEST 2011

CRASHED + SYSTEM does a reboot (es is not restarted automatically)

<no response> Mon Apr 18 19:17:01 CEST 2011
<no response> Mon Apr 18 19:18:01 CEST 2011
<no response> Mon Apr 18 19:19:01 CEST 2011
<no response> Mon Apr 18 19:20:01 CEST 2011

...