elastic.init

$ cat /etc/init/elastic.conf
# ElasticSearch Service
 
description     "ElasticSearch"
 
start on (net-device-up
          and local-filesystems
          and runlevel [2345])
 
stop on runlevel [016]
  
env ES_HOME=/srv/elastic/elasticsearch-0.90.10
env ES_HEAP_SIZE=40g
env ES_MIN_MEM=40g
env ES_MAX_MEM=40g

env DAEMON="${ES_HOME}/bin/elasticsearch"
env DATA_DIR="/data_ssd/elastic/data"
env CONFIG_DIR="${ES_HOME}/config"
env LOG_DIR="/var/log/elastic"
env ES_CLASSPATH=/srv/elastic/hitcount.jar

console output
 
exec su -c "$DAEMON -Xms40G -Xmx40G -f -Des.path.conf=$CONFIG_DIR -Des.path.home=$ES_HOME -Des.path.logs=$LOG_DIR -Des.path.data=$DATA_DIR -Des.index.similarity.default.type=nl.vu.amcat.HitCountSimilarityProvider"

elasticsearch.log

$ sudo tail -f /var/log/elastic/elasticsearch.log
[2014-04-15 10:18:15,227][WARN ][monitor.jvm              ] [Masque] [gc][old][79728][40] duration [1.1m], collections [1]/[1.1m], total [1.1m]/[55.6m], memory [39.8gb]->[39.8gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [33.7mb]->[44mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:20:14,494][WARN ][monitor.jvm              ] [Masque] [gc][old][79729][41] duration [1.9m], collections [1]/[1.9m], total [1.9m]/[57.6m], memory [39.8gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [44mb]->[51.5mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:21:23,949][WARN ][monitor.jvm              ] [Masque] [gc][old][79730][42] duration [1.1m], collections [1]/[1.1m], total [1.1m]/[58.7m], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [51.5mb]->[58.4mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:23:29,236][WARN ][monitor.jvm              ] [Masque] [gc][old][79731][43] duration [2m], collections [1]/[2m], total [2m]/[1h], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [58.4mb]->[61.7mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:24:38,779][WARN ][monitor.jvm              ] [Masque] [gc][old][79732][44] duration [1.1m], collections [1]/[1.1m], total [1.1m]/[1h], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [61.7mb]->[73.4mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:28:06,779][WARN ][monitor.jvm              ] [Masque] [gc][old][79733][46] duration [3.4m], collections [2]/[3.4m], total [3.4m]/[1h], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [73.4mb]->[68.3mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:30:04,377][WARN ][monitor.jvm              ] [Masque] [gc][old][79734][47] duration [1.9m], collections [1]/[1.9m], total [1.9m]/[1.1h], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[599mb]/[599mb]}{[survivor] [68.3mb]->[70.1mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:33:20,989][WARN ][monitor.jvm              ] [Masque] [gc][old][79735][48] duration [1.5m], collections [1]/[1.5m], total [1.5m]/[1.1h], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [599mb]->[598.9mb]/[599mb]}{[survivor] [70.1mb]->[71.2mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 10:36:10,950][WARN ][monitor.jvm              ] [Masque] [gc][old][79736][51] duration [4.5m], collections [3]/[4.5m], total [4.5m]/[1.2h], memory [39.9gb]->[39.9gb]/[39.9gb], all_pools {[young] [598.9mb]->[599mb]/[599mb]}{[survivor] [71.2mb]->[73.4mb]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}
[2014-04-15 11:10:02,924][WARN ][monitor.jvm              ] [Masque] [gc][old][79737][71] duration [27.2m], collections [20]/[33.8m], total [27.2m]/[1.6h], memory [39.9gb]->[39.7gb]/[39.9gb], all_pools {[young] [599mb]->[480.3mb]/[599mb]}{[survivor] [73.4mb]->[0b]/[74.8mb]}{[old] [39.2gb]->[39.2gb]/[39.2gb]}

elasticsearch.yml

wva@amcat-production:~$ grep -v '^#\|^$' /srv/elastic/elasticsearch-0.90.10/config/elasticsearch.yml 
bootstrap.mlockall: true
index.routing.allocation.disable_allocation: false
index.search.slowlog.threshold.query.warn: 10s
index.search.slowlog.threshold.query.info: 5s
index.search.slowlog.threshold.query.debug: 2s
index.search.slowlog.threshold.fetch.warn: 1s
index.search.slowlog.threshold.fetch.info: 800ms
index.search.slowlog.threshold.fetch.debug: 500ms
index.query.bool.max_clause_count: 2048

stats.json

$curl "http://localhost:9200/_nodes/stats/jvm?pretty"
{
  "cluster_name" : "elasticsearch",
  "nodes" : {
    "1M0pZfboSXSM471pBb2qNA" : {
      "timestamp" : 1397553628956,
      "name" : "Masque",
      "transport_address" : "inet[/XX.XX.XX.XX:9300]",
      "hostname" : "amcat-production",
      "jvm" : {
        "timestamp" : 1397553628956,
        "uptime" : "1d",
        "uptime_in_millis" : 87231918,
        "mem" : {
          "heap_used" : "39.9gb",
          "heap_used_in_bytes" : 42847951816,
          "heap_used_percent" : 99,
          "heap_committed" : "39.9gb",
          "heap_committed_in_bytes" : 42871160832,
          "heap_max" : "39.9gb",
          "heap_max_in_bytes" : 42871160832,
          "non_heap_used" : "58.9mb",
          "non_heap_used_in_bytes" : 61835496,
          "non_heap_committed" : "91.9mb",
          "non_heap_committed_in_bytes" : 96411648,
          "pools" : {
            "young" : {
              "used" : "599mb",
              "used_in_bytes" : 628097024,
              "max" : "599mb",
              "max_in_bytes" : 628097024,
              "peak_used" : "599mb",
              "peak_used_in_bytes" : 628097024,
              "peak_max" : "599mb",
              "peak_max_in_bytes" : 628097024
            },
            "survivor" : {
              "used" : "52.7mb",
              "used_in_bytes" : 55314760,
              "max" : "74.8mb",
              "max_in_bytes" : 78512128,
              "peak_used" : "74.8mb",
              "peak_used_in_bytes" : 78512128,
              "peak_max" : "74.8mb",
              "peak_max_in_bytes" : 78512128
            },
            "old" : {
              "used" : "39.2gb",
              "used_in_bytes" : 42164551184,
              "max" : "39.2gb",
              "max_in_bytes" : 42164551680,
              "peak_used" : "39.2gb",
              "peak_used_in_bytes" : 42164551680,
              "peak_max" : "39.2gb",
              "peak_max_in_bytes" : 42164551680
            }
          }
        },
        "threads" : {
          "count" : 142,
          "peak_count" : 167
        },
        "gc" : {
          "collectors" : {
            "young" : {
              "collection_count" : 9462,
              "collection_time" : "20m",
              "collection_time_in_millis" : 1200209
            },
            "old" : {
              "collection_count" : 77,
              "collection_time" : "1.8h",
              "collection_time_in_millis" : 6672456
            }
          }
        },
        "buffer_pools" : {
          "direct" : {
            "count" : 166,
            "used" : "57.4mb",
            "used_in_bytes" : 60286123,
            "total_capacity" : "57.4mb",
            "total_capacity_in_bytes" : 60286123
          },
          "mapped" : {
            "count" : 0,
            "used" : "0b",
            "used_in_bytes" : 0,
            "total_capacity" : "0b",
            "total_capacity_in_bytes" : 0
          }
        }
      }
    }
  }
}