Sample collectd configuration to fetch jetty and jvm metrics for monitoring (and send to graphite)

collectd.conf

LoadPlugin java

<Plugin "java">
   JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"
   LoadPlugin "org.collectd.java.GenericJMX"

   <Plugin "GenericJMX">
   # Garbage collector information
   <MBean "garbage_collector">
     ObjectName "java.lang:type=GarbageCollector,*"
     InstancePrefix "gc-"
     InstanceFrom "name"

     <Value>
       Type "invocations"
       #InstancePrefix ""
       #InstanceFrom ""
       Table false
       Attribute "CollectionCount"
     </Value>
     <Value>
       Type "total_time_in_ms"
       InstancePrefix "collection_time"
       #InstanceFrom ""
       Table false
       Attribute "CollectionTime"
     </Value>
   </MBean>

   # Generic heap/nonheap memory usage.
   <MBean "memory">
     ObjectName "java.lang:type=Memory"
     #InstanceFrom ""
     InstancePrefix "memory"

     # Creates four values: committed, init, max, used
     <Value>
       Type "memory"
       #InstancePrefix ""
       #InstanceFrom ""
       Table true
       Attribute "HeapMemoryUsage"
       InstancePrefix "heap-"
     </Value>

     # Creates four values: committed, init, max, used
     <Value>
       Type "memory"
       #InstancePrefix ""
       #InstanceFrom ""
       Table true
       Attribute "NonHeapMemoryUsage"
       InstancePrefix "nonheap-"
     </Value>
   </MBean>


   <MBean "jetty">
     ObjectName "org.eclipse.jetty.server.handler:type=statisticshandler,id=0"
     InstancePrefix "jetty"
     #InstanceFrom "name"

     <Value>
       Type "response_time"
       Table false
       Attribute "requestTimeMax"
       InstancePrefix "requestTimeMax"
     </Value>
     <Value>
       Type "response_time"
       Table false
       Attribute "requestTimeMean"
       InstancePrefix "requestTimeMean"
     </Value>
     <Value>
       Type "response_time"
       Table false
       Attribute "requestTimeStdDev"
       InstancePrefix "requestTimeStdDev"
     </Value>

     <Value>
       Type "requests"
       Table false
       Attribute "asyncRequestsWaiting"
       InstancePrefix "asyncRequestsWaiting"
     </Value>

     <Value>
       Type "requests"
       Table false
       Attribute "requestsActive"
       InstancePrefix "requestsActive"
     </Value>

     <Value>
       Type "requests"
       Table false
       Attribute "responses1xx"
       InstancePrefix "responses1xx"
     </Value>
     <Value>
       Type "requests"
       Table false
       Attribute "responses2xx"
       InstancePrefix "responses2xx"
     </Value>
     <Value>
       Type "requests"
       Table false
       Attribute "responses3xx"
       InstancePrefix "responses3xx"
     </Value>
     <Value>
       Type "requests"
       Table false
       Attribute "responses4xx"
       InstancePrefix "responses4xx"
     </Value>
     <Value>
       Type "requests"
       Table false
       Attribute "responses5xx"
       InstancePrefix "responses5xx"
     </Value>
   </MBean>


      <Connection>
        ServiceURL "service:jmx:rmi:///jndi/rmi://geonetwork:1099/jmxrmi"
        Host "<java_host>"
        Collect "garbage_collector"
        Collect "memory"
        Collect "jetty"
      </Connection>

   </Plugin>
</Plugin>

gn-collectd.conf

    <MBean "geonetwork_service_manager">
      ObjectName "geonetwork3.jeeves.server.dispatchers:name=*,type=ServiceManager"
      InstancePrefix "gn-sm-"
      InstanceFrom "name"
      <Value>
         Type "response_time"
         Table false
         Attribute "Mean"
      </Value>
      <Value>
         Type "response_time"
         Table false
         Attribute "Max"
      </Value>
      <Value>
         Type "response_time"
         Table false
         Attribute "Min"
      </Value>
    </MBean>

    <MBean "geonetwork_harvester">
      ObjectName "geonetwork3.org.fao.geonet.kernel.harvest.harvester:name=HarvestingErrors,type=AbstractHarvester"
      <Value>
           Type "count"
           Table false
           Attribute "Count"
           InstancePrefix "gn-harvester-errors-count"
         </Value>
    </MBean>


   <MBean "geonetwork_logging">
      ObjectName "geonetwork3.org.apache.log4j:name=*,type=Appender"
      InstancePrefix "gn-logging-"
      InstanceFrom "name"
      <Value>
           Type "derive"
           Table false
           Attribute "MeanRate"
      </Value>
    </MBean>

jmx listing

java.util.logging:type=Logging
org.eclipse.jetty.jmx:name=rmiconnectorserver
org.eclipse.jetty.jmx:type=connectorserver,id=0
org.eclipse.jetty.jmx:context=geonetwork,type=mbeancontainer,id=0
java.nio:type=BufferPool,name=mapped
java.nio:type=BufferPool,name=direct
JMImplementation:type=MBeanServerDelegate
org.eclipse.jetty.util.thread:context=geonetwork,type=scheduledexecutorscheduler,id=0
org.eclipse.jetty.util.thread:type=queuedthreadpool,id=0
org.eclipse.jetty.security:context=geonetwork,type=defaultauthenticatorfactory,id=1
org.eclipse.jetty.security:context=geonetwork,type=constraintsecurityhandler,id=1
geonetwork.org.apache.log4j:type=Appender,name=all
geonetwork.org.apache.log4j:type=Appender,name=errorgeonetwork.org.apache.log4j:type=Appender,name=debug
geonetwork.org.apache.log4j:type=Appender,name=warn
geonetwork.org.apache.log4j:type=Appender,name=fatal
geonetwork.org.apache.log4j:type=Appender,name=info
geonetwork.org.apache.log4j:type=Appender,name=trace
org.eclipse.jetty.annotations:context=geonetwork,type=servletcontainerinitializersstarter,id=1
org.eclipse.jetty.io:context=HTTP/1.1@3f3afe78,type=managedselector,id=2
org.eclipse.jetty.io:context=HTTP/1.1@3f3afe78,type=managedselector,id=3
org.eclipse.jetty.io:context=HTTP/1.1@3f3afe78,type=managedselector,id=0
org.eclipse.jetty.io:context=HTTP/1.1@3f3afe78,type=arraybytebufferpool,id=0
org.eclipse.jetty.io:context=HTTP/1.1@3f3afe78,type=managedselector,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=jsp,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filterholder,name=springSecurityFilterChain,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=wmc,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=MetricsRegistryInitializerFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=listenerholder,id=10
org.eclipse.jetty.servlet:context=geonetwork,type=listenerholder,id=8
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=criticalHealthChecks,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=listenerholder,id=6
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=gn-servlet,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=warningHealthChecks,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=webappMetricsFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=resources,id=9
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=default,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=UrlRewriteFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=HttpProxy,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=expensiveHealthChecks,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=resources,id=5
org.eclipse.jetty.servlet:context=geonetwork,type=listenerholder,id=11
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=springSecurityFilterChain,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=monitor,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=jsp,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filterholder,name=resources,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=expensiveHealthChecks,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filterholder,name=UrlRewriteFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=gn-servlet,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=HttpProxy,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=resources,id=7
org.eclipse.jetty.servlet:context=geonetwork,type=servlethandler,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=default,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=criticalHealthChecks,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=characterEncodingFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=warningHealthChecks,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=wmc,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filterholder,name=characterEncodingFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=monitor,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=listenerholder,id=7
org.eclipse.jetty.servlet:context=geonetwork,type=listenerholder,id=9
org.eclipse.jetty.servlet:context=geonetwork,type=errorpageerrorhandler,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=resources,id=8
org.eclipse.jetty.servlet:context=geonetwork,type=servletmapping,name=mapfish.print,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filtermapping,name=resources,id=6
org.eclipse.jetty.servlet:context=geonetwork,type=filterholder,name=webappMetricsFilter,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=servletholder,name=mapfish.print,id=1
org.eclipse.jetty.servlet:context=geonetwork,type=filterholder,name=MetricsRegistryInitializerFilter,id=1
org.jasig.cas.client.session:context=geonetwork,type=singlesignouthttpsessionlistener,id=1
org.eclipse.jetty.server.session:context=geonetwork,type=sessionhandler,id=1
org.eclipse.jetty.server.session:context=geonetwork,type=hashsessionidmanager,id=0
org.eclipse.jetty.server.session:context=geonetwork,type=hashsessionmanager,id=1
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.serverError
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.ok
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.noContent
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.other
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=activeRequests
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.created
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.notFound
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=responseCodes.badRequest
geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,name=requests
java.util.concurrent:type=concurrenthashmap,id=0
org.eclipse.jetty.deploy.providers:type=webappprovider,id=0
org.eclipse.jetty.server.handler:type=statisticshandler,id=0
org.eclipse.jetty.server.handler:type=contexthandlercollection,id=1
org.eclipse.jetty.server.handler:type=defaulthandler,id=1
org.eclipse.jetty.server.handler:type=handlercollection,id=1
geonetwork.org.fao.geonet.kernel.harvest.harvester:type=AbstractHarvester,name=HarvestingErrors
geonetwork.jeeves.server.dispatchers:type=ServiceManager,name=Output_Xsl_Transform_Timer
geonetwork.jeeves.server.dispatchers:type=ServiceManager,name=Service_Execution_Timer
geonetwork.jeeves.server.dispatchers:type=ServiceManager,name=Gui_services_Timer
java.lang:type=MemoryPool,name=Metaspace
{java.lang:type=MemoryPool,name=PS Old Gen}
{java.lang:type=GarbageCollector,name=PS Scavenge}
{java.lang:type=MemoryPool,name=PS Eden Space}
java.lang:type=Runtime
java.lang:type=Threading
java.lang:type=OperatingSystem
{java.lang:type=MemoryPool,name=Code Cache}
java.lang:type=Compilation
java.lang:type=Mem
oryManager,name=CodeCacheManager
{java.lang:type=MemoryPool,name=Compressed Class Space}
java.lang:type=Memory
{java.lang:type=MemoryPool,name=PS Survivor Space}
java.lang:type=ClassLoading
{java.lang:type=MemoryManager,name=Metaspace Manager}
{java.lang:type=GarbageCollector,name=PS MarkSweep}
com.sun.management:type=DiagnosticCommand
com.sun.management:type=HotSpotDiagnostic
{org.eclipse.jetty.webapp:type=configuration$classlist,id=0}
org.eclipse.jetty.webapp:context=geonetwork,type=webappcontext,id=1
org.eclipse.jetty.server:context=HTTP/1.1@3f3afe78,type=connectorstatistics,id=0
org.eclipse.jetty.server:context=HTTP/1.1@3f3afe78,type=serverconnector,id=0
{org.eclipse.jetty.server:context=HTTP/1.1@3f3afe78,type=abstractconnector$acceptor,id=0}
org.eclipse.jetty.server:type=server,id=0
org.eclipse.jetty.server:context=HTTP/1.1@3f3afe78,type=httpconfiguration,id=0
{org.eclipse.jetty.server:context=HTTP/1.1@3f3afe78,type=serverconnector$serverconnectormanager,id=0}
org.eclipse.jetty.server:context=HTTP/1.1@3f3afe78,type=httpconnectionfactory,id=0
sun.nio.ch:context=HTTP/1.1@3f3afe78,type=serversocketchannelimpl,id=0
org.eclipse.jetty.util.log:type=log,id=0
geonetwork.jeeves.server.resources:type=ResourceManager,name=Resource_is_Open_Timer
geonetwork.jeeves.server.resources:type=ResourceManager,name=Open_Connections_By_ResourceProvider
geonetwork.jeeves.server.resources:type=ResourceManager,name=Max_Connections_By_ResourceProvider
geonetwork.jeeves.server.resources:type=ResourceManager,name=Wait_for_Resource_Timer
{geonetwork.jeeves.server.resources:type=ResourceManager,name=Free files descriptors}
geonetwork.jeeves.server.resources:type=ResourceManager,name=Idle_Connections_By_ResourceProvider
org.eclipse.jetty.deploy:type=deploymentmanager,id=0

using jmxsh

To discover JMX published metrics (or other infos)

jmx_connect -h ... -p1099
jmx_list (quite unreadable)

% jmx_list java.lang
java.lang:type=MemoryPool,name=Metaspace
  {java.lang:type=MemoryPool,name=PS OldGen}
  {java.lang:type=GarbageCollector,name=PS Scavenge}
  {java.lang:type=MemoryPool,name=PS Eden Space}
java.lang:type=Runtime java.lang:type=Threading
java.lang:type=OperatingSystem
  {java.lang:type=MemoryPool,name=Code Cache}
java.lang:type=Compilation
java.lang:type=MemoryManager,name=CodeCacheManager
  {java.lang:type=MemoryPool,name=Compressed Class Space}
java.lang:type=Memory
  {java.lang:type=MemoryPool,name=PS Survivor Space}
java.lang:type=ClassLoading
  {java.lang:type=MemoryManager,name=Metaspace Manager}
  {java.lang:type=GarbageCollector,name=PS MarkSweep}

jmx_get -m object attribute

example:

jmx_get -m java.lang:type=ClassLoading LoadedClassCount
23612

This does not provide the attributes unfortunately :( Better use jconsole for metrics discovery

Then adapt the jmx conf to suit:

   <MBean "<internal name for collected">
     ObjectName "fully.qualified.package:type=MyType(see jconsole ObjectName)"
     InstancePrefix "gc-" // internal use for colectd
     InstanceFrom "name" // internal use for collectd

     <Value>
       Type "name" // internal use for collectd
       Table false // ?
       Attribute "AttributeName" // see jsoncole
     </Value>
      ...
   </MBean>