[tomcat monitoring] JMX + collectd + logstash configuration

collectd.conf

# add the following block:
<Plugin network>
        Server "127.0.0.1"
</Plugin>

jmx.conf

# in /etc/collectd/collectd.conf.d/jmx.conf
<Plugin "java">
  JVMArg "-verbose:jni"
  JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"

  LoadPlugin "org.collectd.java.GenericJMX"

  <Plugin "GenericJMX">
    ################
    # MBean blocks #
    ################
    # Number of classes being loaded.
    <MBean "classes">
      ObjectName "java.lang:type=ClassLoading"
      #InstancePrefix ""
      #InstanceFrom ""

      <Value>
        Type "gauge"
        InstancePrefix "loaded_classes"
        #InstanceFrom ""
        Table false
        Attribute "LoadedClassCount"
      </Value>
    </MBean>

    # Time spent by the JVM compiling or optimizing.
    <MBean "compilation">
      ObjectName "java.lang:type=Compilation"
      #InstancePrefix ""
      #InstanceFrom ""

      <Value>
        Type "total_time_in_ms"
        InstancePrefix "compilation_time"
        #InstanceFrom ""
        Table false
        Attribute "TotalCompilationTime"
      </Value>
    </MBean>

    # Garbage collector information
    <MBean "garbage_collector">
      ObjectName "java.lang:type=GarbageCollector,*"
      InstancePrefix "gc-"
      InstanceFrom "name"
      <Value>
        Type "invocations"
        #InstancePrefix ""
        #InstanceFrom ""
        Table false
        Attribute "CollectionCount"
      </Value>

      <Value>
        Type "total_time_in_ms"
        InstancePrefix "collection_time"
        #InstanceFrom ""
        Table false
        Attribute "CollectionTime"
      </Value>

#      # Not that useful, therefore commented out.
#      <Value>
#        Type "threads"
#        #InstancePrefix ""
#        #InstanceFrom ""
#        Table false
#        # Demonstration how to access composite types
#        Attribute "LastGcInfo.GcThreadCount"
#      </Value>
    </MBean>

    ######################################
    # Define the "jmx_memory" type as:   #
    #   jmx_memory  value:GAUGE:0:U      #
    # See types.db(5) for details.       #
    ######################################

    # Generic heap/nonheap memory usage.
    <MBean "memory">
      ObjectName "java.lang:type=Memory"
      #InstanceFrom ""
      InstancePrefix "memory"

      # Creates four values: committed, init, max, used
      <Value>
        Type "jmx_memory"
        #InstancePrefix ""
        #InstanceFrom ""
        Table true
        Attribute "HeapMemoryUsage"
        InstancePrefix "heap-"
      </Value>
      # Creates four values: committed, init, max, used
      <Value>
        Type "jmx_memory"
        #InstancePrefix ""
        #InstanceFrom ""
        Table true
        Attribute "NonHeapMemoryUsage"
        InstancePrefix "nonheap-"
      </Value>
    </MBean>

    # Memory usage by memory pool.
    <MBean "memory_pool">
      ObjectName "java.lang:type=MemoryPool,*"
      InstancePrefix "memory_pool-"
      InstanceFrom "name"

      <Value>
        Type "jmx_memory"
        #InstancePrefix ""
        #InstanceFrom ""
        Table true
        Attribute "Usage"
      </Value>
    </MBean>

    ### MBeans by Catalina / Tomcat ###
    # The global request processor (summary for each request processor)
    <MBean "catalina/global_request_processor">
      ObjectName "Catalina:type=GlobalRequestProcessor,*"
      InstancePrefix "request_processor-"
      InstanceFrom "name"

      <Value>
        Type "io_octets"
        InstancePrefix "global"
        #InstanceFrom ""
        Table false
        Attribute "bytesReceived"
        Attribute "bytesSent"
      </Value>

      <Value>
        Type "total_requests"
        InstancePrefix "global"
        #InstanceFrom ""
        Table false
        Attribute "requestCount"
      </Value>

      <Value>
        Type "total_time_in_ms"
        InstancePrefix "global-processing"
        #InstanceFrom ""
        Table false
        Attribute "processingTime"
      </Value>
    </MBean>

    # Details for each  request processor
    <MBean "catalina/detailed_request_processor">
      ObjectName "Catalina:type=RequestProcessor,*"
      InstancePrefix "request_processor-"
      InstanceFrom "worker"

      <Value>
        Type "io_octets"
        #InstancePrefix ""
        InstanceFrom "name"
        Table false
        Attribute "bytesReceived"
        Attribute "bytesSent"
      </Value>

      <Value>
        Type "total_requests"
        #InstancePrefix ""
        InstanceFrom "name"
        Table false
        Attribute "requestCount"
      </Value>

      <Value>
        Type "total_time_in_ms"
        InstancePrefix "processing-"
        InstanceFrom "name"
        Table false
        Attribute "processingTime"
      </Value>
    </MBean>

    # Thread pool
    <MBean "catalina/thread_pool">
      ObjectName "Catalina:type=ThreadPool,*"
      InstancePrefix "request_processor-"
      InstanceFrom "name"

      <Value>
        Type "threads"
        InstancePrefix "total"
        #InstanceFrom ""
        Table false
        Attribute "currentThreadCount"
      </Value>

      <Value>
        Type "threads"
        InstancePrefix "running"
        #InstanceFrom ""
        Table false
        Attribute "currentThreadsBusy"
      </Value>
    </MBean>

    #####################
    # Connection blocks #
    #####################
    <Connection>
      ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"
      # host can be an arbitrary value (localhost-geoserver0, ...)
      Host "localhost"
      Collect "classes"
      Collect "compilation"
      Collect "garbage_collector"
      Collect "memory"
      Collect "memory_pool"
    </Connection>
  </Plugin>
</Plugin>

log4j.properties

# sample log4j using a socket appender
log4j.logger.org.georchestra.security.statistics-common=INFO, NETWORKSOCKET

log4j.appender.NETWORKSOCKET.RemoteHost=localhost
log4j.logger.org.georchestra.security.statistics-common=INFO,NETWORKSOCKET
log4j.appender.NETWORKSOCKET=org.apache.log4j.net.SocketAppender
log4j.appender.NETWORKSOCKET.Port=4712

logstash.conf

# sample logstash / georchestra configuration file
input {
        log4j {
                mode => server
                port => 4712
                type => "sp-common-logs"
        }
        udp {
                port => 25826
                buffer_size => 1452
                type => "collectd"
                codec => collectd { }
        }
}
filter {
        if [type] == "sp-common-logs" {
                grok { match => [ "message", "%{COMMONAPACHELOG} (?<time.needed>(%{BASE10NUM}))" ] }
                mutate {
                        convert => {
                                "time.needed" => "float"
                                "response"    => "integer"
                                "bytes"       => "integer"
                        }
                }
        }
}
output {
        elasticsearch {}
}

setenv-local.sh

# /srv/tomcat/geoserver0/bin/setenv-local.sh
export ADD_JAVA_OPTS="$ADD_JAVA_OPTS -Dcom.sun.management.jmxremote    \
-Dcom.sun.management.jmxremote.port=9010                               \
-Dcom.sun.management.jmxremote.local.only=true                         \
-Dcom.sun.management.jmxremote.authenticate=false                      \
-Dcom.sun.management.jmxremote.ssl=false"

types.db

# in /usr/share/collectd/types.db
# Add:
jmx_memory              value:GAUGE:0:U

For Java 8 and Amazon Cloudwatch Agent with Collectd support, I had to comment-out the "nonheap-" section to avoid the following error:

2019-04-16T09:37:31Z W! Value cannot be negative: -1