ebuildy/install.sh

## install.sh
#!/bin/sh

sudo apt-get install snmpd xinetd
#sudo nano /etc/snmp/snmpd.conf

sudo curl -o /usr/bin/observium_agent https://gist.githubusercontent.com/ebuildy/6aab534af385ab535b99/raw/c6665fbcc086ca953c08b9b85189d67d6bfd5387/observium_agent
sudo curl -o /etc/xinetd.d/observium_agent_xinetd https://gist.githubusercontent.com/ebuildy/6aab534af385ab535b99/raw/ec27ea4f5d2b11d6b0f35296dbdbbca5f2cc3ee1/observium_agent_xinetd

sudo chmod a+x /usr/bin/observium_agent

sudo service xinetd restart
sudo mkdir /usr/lib/observium_agent
sudo mkdir /usr/lib/observium_agent/local

## observium_agent
#!/bin/bash
#                _                         _
#           ___ | |__  ___  ___ _ ____   _(_)_   _ _ __ ___
#          / _ \| '_ \/ __|/ _ \ '__\ \ / / | | | | '_ ` _ \
#         | (_) | |_) \__ \  __/ |   \ V /| | |_| | | | | | |
#          \___/|_.__/|___/\___|_|    \_/ |_|\__,_|_| |_| |_|
#                                                 Linux Agent
#  Copyright Adam Armstrong 2013
#
#  Based on Check_MK from Mathias Kettner <mk@mathias-kettner.de>
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# ails.  You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

# Remove locale settings to eliminate localized outputs where possible
export LC_ALL=C
unset LANG

export AGENT_LIBDIR="/usr/lib/observium_agent"
export AGENT_CONFDIR="/etc/observium"

# Make sure, locally installed binaries are found
PATH=$PATH:/usr/local/bin

# All executables in PLUGINSDIR will simply be executed and their
# ouput appended to the output of the agent. Plugins define their own
# sections and must output headers with '<<<' and '>>>'
PLUGINSDIR=$AGENT_LIBDIR/plugins

# All executables in LOCALDIR will by executabled and their
# output inserted into the section <<<local>>>. Please
# refer to online documentation for details about local checks.
LOCALDIR=$AGENT_LIBDIR/local


# close standard input (for security reasons) and stderr
if [ "$1" = -d ]
then
    set -xv
else
    exec <&- 2>/dev/null
fi

echo '<<<Observium>>>'
echo Version: 1.0.1
echo AgentOS: linux
echo PluginsDirectory: $PLUGINSDIR
echo LocalDirectory: $LOCALDIR
echo AgentDirectory: $AGENT_CONFDIR

# If we are called via xinetd, try to find only_from configuration
if [ -n "$REMOTE_HOST" ]
then
    echo -n 'OnlyFrom: '
    echo $(sed -n '/^service[[:space:]]*observium_agent/,/}/s/^[[:space:]]*only_from[[:space:]]*=[[:space:]]*\(.*\)/\1/p' /etc/xinetd.d/* | head -n1)
fi

# Show filesystems. -P Prevents wrapping long mount points
# Hide NFS mounts to prevent hanging
echo '<<<df>>>'
df -PTlk -x smbfs -x tmpfs -x cifs -x iso9660 -x udf -x nfsv4 | sed 1d
# VMWare shows its own filesystems with 'vdf'. Just one
# problem: it outputs not 7 but only 6 columns
if which vdf > /dev/null
then
   vdf -P | grep ^/vmfs/volumes | sed 's/ / vmfs /'
fi

# Check NFS mounts by accessing them with stat -f (System
# call statfs()). If this lasts more then 2 seconds we
# consider it as hanging. We need waitmax.
if type waitmax >/dev/null
then
    STAT_VERSION=$(stat --version | head -1 | cut -d" " -f4)
    STAT_BROKE="5.3.0"

    echo '<<<nfsmounts>>>'
    sed -n '/ nfs /s/[^ ]* \([^ ]*\) .*/\1/p' < /proc/mounts |
        while read MP
	do
	 if [ $STAT_VERSION != $STAT_BROKE ]; then
	    waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" || \
		echo "$MP hanging 0 0 0 0"
	 else
	    waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" && \
	    printf '\n'|| echo "$MP hanging 0 0 0 0"
	 fi
	done
fi

# Check mount options. Filesystems may switch to 'ro' in case
# of a read error.
echo '<<<mounts>>>'
grep ^/dev < /proc/mounts

# processes including username, without kernel processes
echo '<<<ps>>>'
ps ax -o user,vsz,rss,pcpu,command --columns 10000 | sed -e 1d -e 's/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4) /'

# Memory Usage
echo '<<<mem>>>'
egrep -v '^Swap:|^Mem:|total:' < /proc/meminfo

# Load and active processes.
echo '<<<cpu>>>'
echo "$(cat /proc/loadavg) $(grep -E '^CPU|^processor' < /proc/cpuinfo | wc -l)"

# Uptime
echo '<<<uptime>>>'
cat /proc/uptime

# Network interfaces (Link, Autoneg, Speed)
# This requires ethtool
if which ethtool > /dev/null
then
  echo '<<<netif>>>'
  for eth in $(cat /proc/net/dev | sed -rn -e 's/[[:space:]]*//g' -e  '/ *([^:]):.*/s//\1/p' | egrep -vx '(lo|sit.*)')
  do
    echo $eth $(ethtool $eth | egrep '(Speed|Duplex|Link detected|Auto-negotiation):' | cut -d: -f2 | sed 's/ *//g')
  done
fi

# New variant: Information about speed and state in one section
echo '<<<lnx_if:sep(58)>>>'
sed 1,2d /proc/net/dev
if which ethtool > /dev/null
then
    for eth in $(sed -e 1,2d < /proc/net/dev | cut -d':' -f1)
    do
      echo "[$eth]"
      ethtool $eth | egrep '(Speed|Duplex|Link detected|Auto-negotiation):'
    done
fi

# Number of TCP connections in the various states
echo '<<<tcp_conn_stats>>>'
netstat -nt | awk ' /^tcp/ { c[$6]++; } END { for (x in c) { print x, c[x]; } }'

# Disk and RAID status of LSI Controllers, if available
if which cfggen > /dev/null ; then
   echo '<<<lsi>>>'
   cfggen 0 DISPLAY | egrep '(Target ID|State|Volume ID|Status of volume)[[:space:]]*:' | sed -e 's/ *//g' -e 's/:/ /'
fi

# Multipath devices.
if which multipath >/dev/null ; then
    echo '<<<multipath>>>'
    multipath -l
fi

# Soft-RAID
echo '<<<md>>>'
cat /proc/mdstat

# Disk performance counters
echo '<<<diskstat>>>'
date +%s
egrep ' (x?[shv]d[a-z]*|cciss/c[0-9]+d[0-9]+) ' < /proc/diskstats

# Kernel performance counters
echo '<<<kernel>>>'
date +%s
cat /proc/vmstat /proc/stat

# Network performance counters (Packets, collisions, etc)
echo '<<<netctr>>>'
# Exact timestamp because counters depend upon the time
date +%s
sed -e 1,2d -e 's/:/ /g' < /proc/net/dev

# IPMI data via ipmi-sensors (of freeipmi). Please make sure, that if you
# have installed freeipmi that IPMI is really support by your hardware.
# The agent tries to avoid hanging forever by setting a limit of 300 seconds
# for the first run (where the cache is created). If ipmi-sensors runs into
# that timeout, it leaves and empty cache file. We skip this check forever
# if we find that empty cache file.
sdrcache=/var/cache/.freeipmi/sdr-cache/sdr-cache-$(hostname).127.0.0.1
if which ipmi-sensors >/dev/null && [ ! -e "$sdrcache" -o -s "$sdrcache" ]
then
    echo '<<<ipmi_sensors>>>'
    # No cache file existing? => Impose a high time limit. We do not suffice
    # in creating the cache we most probably run on a hardware where this tool
    # is hanging forever. We make sure that we never try again in that case!
    if [ ! -e "$sdrcache" ]
    then
        WAITMAX="waitmax 300"
    elif tail --bytes 2 < "$sdrcache" | od -t x2 | grep -q 0a0a
    then
        WAITMAX="waitmax 3"
    else
        # Cache file corrupt. Must end with two linefeeds.
        rm -f $sdrcache
	WAITMAX=
    fi
    # Newer ipmi-sensors version have new output format; Legacy format can be used
    if ipmi-sensors --help | grep -q legacy-output; then
        IPMI_FORMAT="--legacy-output"
    else
        IPMI_FORMAT=""
    fi
    # Aquire lock with flock in order to avoid multiple runs of ipmi-sensors
    # in case of parallel or overlapping calls of the agent.
    (
        flock -n 200 --wait 60
        # At least with ipmi-sensoirs 0.7.16 this group is Power_Unit instead of "Power Unit"
        for class in Temperature Power_Unit Fan
        do
            $WAITMAX ipmi-sensors $IPMI_FORMAT --sdr-cache-directory /var/cache -g "$class" | sed -e 's/ /_/g' -e 's/:_\?/ /g' -e 's@ \([^(]*\)_(\([^)]*\))@ \2_\1@'
            # In case of a timeout immediately leave loop.
        if [ $? = 255 ] ; then break ; fi
            WAITMAX="waitmax 3"
        done
    ) 200>>"$sdrcache"
fi

# State of LSI MegaRAID controller via MegaCli. You can download that tool from:
# http://www.lsi.com/downloads/Public/MegaRAID%20Common%20Files/8.02.16_MegaCLI.zip

if which MegaCli >/dev/null ; then
    echo '<<<megaraid_pdisks>>>'
    for part in $(MegaCli -EncInfo -aALL -NoLog < /dev/null \
        | sed -rn 's/:/ /g; s/[[:space:]]+/ /g; s/^ //; s/ $//; s/Number of enclosures on adapter ([0-9]+).*/adapter \1/g; /^(Enclosure|Device ID|adapter) [0-9]+$/ p'); do
        [ $part = adapter ] && echo ""
        [ $part = 'Enclosure' ] && echo -ne "\ndev2enc"
        echo -n " $part"
    done
    echo
    MegaCli -PDList -aALL -NoLog < /dev/null | egrep 'Enclosure|Raw Size|Slot Number|Device Id|Firmware state|Inquiry|Adapter'
    echo '<<<megaraid_ldisks>>>'
    MegaCli -LDInfo -Lall -aALL -NoLog < /dev/null | egrep 'Size|State|Number|Adapter|Virtual'
    echo '<<<megaraid_bbu>>>'
    MegaCli -AdpBbuCmd -GetBbuStatus -aALL -NoLog < /dev/null | grep -v Exit
fi

# 3WARE disk controller (by Radoslaw Bak)
if which tw_cli > /dev/null ; then
    for C in $(tw_cli show | awk 'NR < 4 { next } { print $1 }'); do
        echo '<<<3ware_info>>>'
        tw_cli /$C show all | egrep 'Model =|Firmware|Serial'
        echo '<<<3ware_disks>>>'
        tw_cli /$C show drivestatus | egrep 'p[0-9]' | sed "s/^/$C\//"
        echo '<<<3ware_units>>>'
        tw_cli /$C show unitstatus | egrep 'u[0-9]' | sed "s/^/$C\//"
    done
fi

if which vcbVmName > /dev/null 2>&1 ; then
   echo '<<<vmware_state>>>'
   vcbVmName -s any
fi

if which ntpq > /dev/null 2>&1 ; then
   echo '<<<ntp>>>'
   # remote heading, make first column space separated
   waitmax 2 ntpq -p | sed -e 1,2d -e 's/^\(.\)/\1 /' -e 's/^ /%/'
fi

if [ -e /proc/drbd ]; then
  echo '<<<drbd>>>'
  cat /proc/drbd
fi

# Status of CUPS printer queues
if which lpstat > /dev/null 2>&1; then
   if pgrep cups > /dev/null 2>&1; then
       echo '<<<cups_queues>>>'
       waitmax 3 lpstat -p
       echo '---'
       waitmax 3 lpstat -o|sort
   fi
fi

# Heartbeat monitoring
if which cl_status > /dev/null 2>&1; then
  # Different handling for heartbeat clusters with and without CRM
  # for the resource state
  if [ -S /var/run/heartbeat/crm/cib_ro -o -S /var/run/crm/cib_ro ]; then
    echo '<<<heartbeat_crm>>>'
    crm_mon -1 -r | grep -v ^$ | sed '/^\sResource Group:/,$ s/^\s//; s/^\s/_/g'
  else
    echo '<<<heartbeat_rscstatus>>>'
    cl_status rscstatus
  fi

  echo '<<<heartbeat_nodes>>>'
  for NODE in $(cl_status listnodes); do
    if [ $NODE != $(echo $HOSTNAME | tr 'A-Z' 'a-z') ]; then
      STATUS=$(cl_status nodestatus $NODE)
      echo -n "$NODE $STATUS"
      for LINK in $(cl_status listhblinks $NODE 2>/dev/null); do
        echo -n " $LINK $(cl_status hblinkstatus $NODE $LINK)"
      done
      echo
    fi
  done
fi

# Postfix mailqueue monitoring
#
# Only handle mailq when postfix user is present. The mailq command is also
# available when postfix is not installed. But it produces different outputs
# which are not handled by the check at the moment. So try to filter out the
# systems not using postfix by searching for the postfix user.a
#
# Cannot take the whole outout. This could produce several MB of agent output
# on blocking queues.
# Only handle the last 6 lines (includes the summary line at the bottom and
# the last message in the queue. The last message is not used at the moment
# but it could be used to get the timestamp of the last message.
if which mailq >/dev/null 2>&1 && getent passwd postfix >/dev/null 2>&1; then
  echo '<<<postfix_mailq>>>'
  mailq | tail -n 6
fi

# Check status of OMD sites
if type omd >/dev/null
then
   echo '<<<omd_status>>>'
   omd status --bare --auto
fi

# Einbinden von lokalen Plugins, die eine eigene Sektion ausgeben
if cd $PLUGINSDIR
then
  for skript in $(ls)
  do
    if [ -x "$skript" ] ; then
        ./$skript
    fi
  done
fi

# Lokale Einzelchecks
echo '<<<local>>>'
if cd $LOCALDIR
then
  for skript in $(ls|grep -v '~$')
  do
    if [ -x "$skript" ] ; then
        ./$skript
    fi
  done
fi

## observium_agent_xinetd
service observium_agent
{
        type           = UNLISTED
        port           = 36602
        socket_type    = stream
        protocol       = tcp
        wait           = no
        user           = root
        server         = /usr/bin/observium_agent

        # configure the IPv[4|6] address(es) of your Observium server here:
        # only_from      = 127.0.0.1 ::1

        # Don't be too verbose. Don't log every check. This might be
        # commented out for debugging. If this option is commented out
        # the default options will be used for this service.
        log_on_success =

        disable        = no
}
	#!/bin/sh

	sudo apt-get install snmpd xinetd
	#sudo nano /etc/snmp/snmpd.conf

	sudo curl -o /usr/bin/observium_agent https://gist.githubusercontent.com/ebuildy/6aab534af385ab535b99/raw/c6665fbcc086ca953c08b9b85189d67d6bfd5387/observium_agent
	sudo curl -o /etc/xinetd.d/observium_agent_xinetd https://gist.githubusercontent.com/ebuildy/6aab534af385ab535b99/raw/ec27ea4f5d2b11d6b0f35296dbdbbca5f2cc3ee1/observium_agent_xinetd

	sudo chmod a+x /usr/bin/observium_agent

	sudo service xinetd restart
	sudo mkdir /usr/lib/observium_agent
	sudo mkdir /usr/lib/observium_agent/local
	#!/bin/bash
	# _ _
	# ___ \| \|__ ___ ___ _ ____ _(_)_ _ _ __ ___
	# / _ \\| '_ \/ __\|/ _ \ '__\ \ / / \| \| \| \| '_ ` _ \
	# \| (_) \| \|_) \__ \ __/ \| \ V /\| \| \|_\| \| \| \| \| \| \|
	# \___/\|_.__/\|___/\___\|_\| \_/ \|_\|\__,_\|_\| \|_\| \|_\|
	# Linux Agent
	# Copyright Adam Armstrong 2013
	#
	# Based on Check_MK from Mathias Kettner <mk@mathias-kettner.de>
	#
	# check_mk is free software; you can redistribute it and/or modify it
	# under the terms of the GNU General Public License as published by
	# the Free Software Foundation in version 2. check_mk is distributed
	# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with-
	# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A
	# PARTICULAR PURPOSE. See the GNU General Public License for more de-
	# ails. You should have received a copy of the GNU General Public
	# License along with GNU Make; see the file COPYING. If not, write
	# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
	# Boston, MA 02110-1301 USA.

	# Remove locale settings to eliminate localized outputs where possible
	export LC_ALL=C
	unset LANG

	export AGENT_LIBDIR="/usr/lib/observium_agent"
	export AGENT_CONFDIR="/etc/observium"

	# Make sure, locally installed binaries are found
	PATH=$PATH:/usr/local/bin

	# All executables in PLUGINSDIR will simply be executed and their
	# ouput appended to the output of the agent. Plugins define their own
	# sections and must output headers with '<<<' and '>>>'
	PLUGINSDIR=$AGENT_LIBDIR/plugins

	# All executables in LOCALDIR will by executabled and their
	# output inserted into the section <<<local>>>. Please
	# refer to online documentation for details about local checks.
	LOCALDIR=$AGENT_LIBDIR/local


	# close standard input (for security reasons) and stderr
	if [ "$1" = -d ]
	then
	set -xv
	else
	exec <&- 2>/dev/null
	fi

	echo '<<<Observium>>>'
	echo Version: 1.0.1
	echo AgentOS: linux
	echo PluginsDirectory: $PLUGINSDIR
	echo LocalDirectory: $LOCALDIR
	echo AgentDirectory: $AGENT_CONFDIR

	# If we are called via xinetd, try to find only_from configuration
	if [ -n "$REMOTE_HOST" ]
	then
	echo -n 'OnlyFrom: '
	echo $(sed -n '/^service[[:space:]]observium_agent/,/}/s/^[[:space:]]only_from[[:space:]]=[[:space:]]\(.\)/\1/p' /etc/xinetd.d/ \| head -n1)
	fi

	# Show filesystems. -P Prevents wrapping long mount points
	# Hide NFS mounts to prevent hanging
	echo '<<<df>>>'
	df -PTlk -x smbfs -x tmpfs -x cifs -x iso9660 -x udf -x nfsv4 \| sed 1d
	# VMWare shows its own filesystems with 'vdf'. Just one
	# problem: it outputs not 7 but only 6 columns
	if which vdf > /dev/null
	then
	vdf -P \| grep ^/vmfs/volumes \| sed 's/ / vmfs /'
	fi

	# Check NFS mounts by accessing them with stat -f (System
	# call statfs()). If this lasts more then 2 seconds we
	# consider it as hanging. We need waitmax.
	if type waitmax >/dev/null
	then
	STAT_VERSION=$(stat --version \| head -1 \| cut -d" " -f4)
	STAT_BROKE="5.3.0"

	echo '<<<nfsmounts>>>'
	sed -n '/ nfs /s/[^ ]* \([^ ]\) ./\1/p' < /proc/mounts \|
	while read MP
	do
	if [ $STAT_VERSION != $STAT_BROKE ]; then
	waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" \|\| \
	echo "$MP hanging 0 0 0 0"
	else
	waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" && \
	printf '\n'\|\| echo "$MP hanging 0 0 0 0"
	fi
	done
	fi

	# Check mount options. Filesystems may switch to 'ro' in case
	# of a read error.
	echo '<<<mounts>>>'
	grep ^/dev < /proc/mounts

	# processes including username, without kernel processes
	echo '<<<ps>>>'
	ps ax -o user,vsz,rss,pcpu,command --columns 10000 \| sed -e 1d -e 's/ \([^ ]\) \([^ ]\) \([^ ]\) \([^ ]\) */(\1,\2,\3,\4) /'

	# Memory Usage
	echo '<<<mem>>>'
	egrep -v '^Swap:\|^Mem:\|total:' < /proc/meminfo

	# Load and active processes.
	echo '<<<cpu>>>'
	echo "$(cat /proc/loadavg) $(grep -E '^CPU\|^processor' < /proc/cpuinfo \| wc -l)"

	# Uptime
	echo '<<<uptime>>>'
	cat /proc/uptime

	# Network interfaces (Link, Autoneg, Speed)
	# This requires ethtool
	if which ethtool > /dev/null
	then
	echo '<<<netif>>>'
	for eth in $(cat /proc/net/dev \| sed -rn -e 's/[[:space:]]//g' -e '/ ([^:]):./s//\1/p' \| egrep -vx '(lo\|sit.)')
	do
	echo $eth $(ethtool $eth \| egrep '(Speed\|Duplex\|Link detected\|Auto-negotiation):' \| cut -d: -f2 \| sed 's/ *//g')
	done
	fi

	# New variant: Information about speed and state in one section
	echo '<<<lnx_if:sep(58)>>>'
	sed 1,2d /proc/net/dev
	if which ethtool > /dev/null
	then
	for eth in $(sed -e 1,2d < /proc/net/dev \| cut -d':' -f1)
	do
	echo "[$eth]"
	ethtool $eth \| egrep '(Speed\|Duplex\|Link detected\|Auto-negotiation):'
	done
	fi

	# Number of TCP connections in the various states
	echo '<<<tcp_conn_stats>>>'
	netstat -nt \| awk ' /^tcp/ { c[$6]++; } END { for (x in c) { print x, c[x]; } }'

	# Disk and RAID status of LSI Controllers, if available
	if which cfggen > /dev/null ; then
	echo '<<<lsi>>>'
	cfggen 0 DISPLAY \| egrep '(Target ID\|State\|Volume ID\|Status of volume)[[:space:]]:' \| sed -e 's/ //g' -e 's/:/ /'
	fi

	# Multipath devices.
	if which multipath >/dev/null ; then
	echo '<<<multipath>>>'
	multipath -l
	fi

	# Soft-RAID
	echo '<<<md>>>'
	cat /proc/mdstat

	# Disk performance counters
	echo '<<<diskstat>>>'
	date +%s
	egrep ' (x?[shv]d[a-z]*\|cciss/c[0-9]+d[0-9]+) ' < /proc/diskstats

	# Kernel performance counters
	echo '<<<kernel>>>'
	date +%s
	cat /proc/vmstat /proc/stat

	# Network performance counters (Packets, collisions, etc)
	echo '<<<netctr>>>'
	# Exact timestamp because counters depend upon the time
	date +%s
	sed -e 1,2d -e 's/:/ /g' < /proc/net/dev

	# IPMI data via ipmi-sensors (of freeipmi). Please make sure, that if you
	# have installed freeipmi that IPMI is really support by your hardware.
	# The agent tries to avoid hanging forever by setting a limit of 300 seconds
	# for the first run (where the cache is created). If ipmi-sensors runs into
	# that timeout, it leaves and empty cache file. We skip this check forever
	# if we find that empty cache file.
	sdrcache=/var/cache/.freeipmi/sdr-cache/sdr-cache-$(hostname).127.0.0.1
	if which ipmi-sensors >/dev/null && [ ! -e "$sdrcache" -o -s "$sdrcache" ]
	then
	echo '<<<ipmi_sensors>>>'
	# No cache file existing? => Impose a high time limit. We do not suffice
	# in creating the cache we most probably run on a hardware where this tool
	# is hanging forever. We make sure that we never try again in that case!
	if [ ! -e "$sdrcache" ]
	then
	WAITMAX="waitmax 300"
	elif tail --bytes 2 < "$sdrcache" \| od -t x2 \| grep -q 0a0a
	then
	WAITMAX="waitmax 3"
	else
	# Cache file corrupt. Must end with two linefeeds.
	rm -f $sdrcache
	WAITMAX=
	fi
	# Newer ipmi-sensors version have new output format; Legacy format can be used
	if ipmi-sensors --help \| grep -q legacy-output; then
	IPMI_FORMAT="--legacy-output"
	else
	IPMI_FORMAT=""
	fi
	# Aquire lock with flock in order to avoid multiple runs of ipmi-sensors
	# in case of parallel or overlapping calls of the agent.
	(
	flock -n 200 --wait 60
	# At least with ipmi-sensoirs 0.7.16 this group is Power_Unit instead of "Power Unit"
	for class in Temperature Power_Unit Fan
	do
	$WAITMAX ipmi-sensors $IPMI_FORMAT --sdr-cache-directory /var/cache -g "$class" \| sed -e 's/ /_/g' -e 's/:_\?/ /g' -e 's@ \([^(]\)_(\([^)]\))@ \2_\1@'
	# In case of a timeout immediately leave loop.
	if [ $? = 255 ] ; then break ; fi
	WAITMAX="waitmax 3"
	done
	) 200>>"$sdrcache"
	fi

	# State of LSI MegaRAID controller via MegaCli. You can download that tool from:
	# http://www.lsi.com/downloads/Public/MegaRAID%20Common%20Files/8.02.16_MegaCLI.zip

	if which MegaCli >/dev/null ; then
	echo '<<<megaraid_pdisks>>>'
	for part in $(MegaCli -EncInfo -aALL -NoLog < /dev/null \
	\| sed -rn 's/:/ /g; s/[[:space:]]+/ /g; s/^ //; s/ $//; s/Number of enclosures on adapter ([0-9]+).*/adapter \1/g; /^(Enclosure\|Device ID\|adapter) [0-9]+$/ p'); do
	[ $part = adapter ] && echo ""
	[ $part = 'Enclosure' ] && echo -ne "\ndev2enc"
	echo -n " $part"
	done
	echo
	MegaCli -PDList -aALL -NoLog < /dev/null \| egrep 'Enclosure\|Raw Size\|Slot Number\|Device Id\|Firmware state\|Inquiry\|Adapter'
	echo '<<<megaraid_ldisks>>>'
	MegaCli -LDInfo -Lall -aALL -NoLog < /dev/null \| egrep 'Size\|State\|Number\|Adapter\|Virtual'
	echo '<<<megaraid_bbu>>>'
	MegaCli -AdpBbuCmd -GetBbuStatus -aALL -NoLog < /dev/null \| grep -v Exit
	fi

	# 3WARE disk controller (by Radoslaw Bak)
	if which tw_cli > /dev/null ; then
	for C in $(tw_cli show \| awk 'NR < 4 { next } { print $1 }'); do
	echo '<<<3ware_info>>>'
	tw_cli /$C show all \| egrep 'Model =\|Firmware\|Serial'
	echo '<<<3ware_disks>>>'
	tw_cli /$C show drivestatus \| egrep 'p[0-9]' \| sed "s/^/$C\//"
	echo '<<<3ware_units>>>'
	tw_cli /$C show unitstatus \| egrep 'u[0-9]' \| sed "s/^/$C\//"
	done
	fi

	if which vcbVmName > /dev/null 2>&1 ; then
	echo '<<<vmware_state>>>'
	vcbVmName -s any
	fi

	if which ntpq > /dev/null 2>&1 ; then
	echo '<<<ntp>>>'
	# remote heading, make first column space separated
	waitmax 2 ntpq -p \| sed -e 1,2d -e 's/^\(.\)/\1 /' -e 's/^ /%/'
	fi

	if [ -e /proc/drbd ]; then
	echo '<<<drbd>>>'
	cat /proc/drbd
	fi

	# Status of CUPS printer queues
	if which lpstat > /dev/null 2>&1; then
	if pgrep cups > /dev/null 2>&1; then
	echo '<<<cups_queues>>>'
	waitmax 3 lpstat -p
	echo '---'
	waitmax 3 lpstat -o\|sort
	fi
	fi

	# Heartbeat monitoring
	if which cl_status > /dev/null 2>&1; then
	# Different handling for heartbeat clusters with and without CRM
	# for the resource state
	if [ -S /var/run/heartbeat/crm/cib_ro -o -S /var/run/crm/cib_ro ]; then
	echo '<<<heartbeat_crm>>>'
	crm_mon -1 -r \| grep -v ^$ \| sed '/^\sResource Group:/,$ s/^\s//; s/^\s/_/g'
	else
	echo '<<<heartbeat_rscstatus>>>'
	cl_status rscstatus
	fi

	echo '<<<heartbeat_nodes>>>'
	for NODE in $(cl_status listnodes); do
	if [ $NODE != $(echo $HOSTNAME \| tr 'A-Z' 'a-z') ]; then
	STATUS=$(cl_status nodestatus $NODE)
	echo -n "$NODE $STATUS"
	for LINK in $(cl_status listhblinks $NODE 2>/dev/null); do
	echo -n " $LINK $(cl_status hblinkstatus $NODE $LINK)"
	done
	echo
	fi
	done
	fi

	# Postfix mailqueue monitoring
	#
	# Only handle mailq when postfix user is present. The mailq command is also
	# available when postfix is not installed. But it produces different outputs
	# which are not handled by the check at the moment. So try to filter out the
	# systems not using postfix by searching for the postfix user.a
	#
	# Cannot take the whole outout. This could produce several MB of agent output
	# on blocking queues.
	# Only handle the last 6 lines (includes the summary line at the bottom and
	# the last message in the queue. The last message is not used at the moment
	# but it could be used to get the timestamp of the last message.
	if which mailq >/dev/null 2>&1 && getent passwd postfix >/dev/null 2>&1; then
	echo '<<<postfix_mailq>>>'
	mailq \| tail -n 6
	fi

	# Check status of OMD sites
	if type omd >/dev/null
	then
	echo '<<<omd_status>>>'
	omd status --bare --auto
	fi

	# Einbinden von lokalen Plugins, die eine eigene Sektion ausgeben
	if cd $PLUGINSDIR
	then
	for skript in $(ls)
	do
	if [ -x "$skript" ] ; then
	./$skript
	fi
	done
	fi

	# Lokale Einzelchecks
	echo '<<<local>>>'
	if cd $LOCALDIR
	then
	for skript in $(ls\|grep -v '~$')
	do
	if [ -x "$skript" ] ; then
	./$skript
	fi
	done
	fi
	service observium_agent
	{
	type = UNLISTED
	port = 36602
	socket_type = stream
	protocol = tcp
	wait = no
	user = root
	server = /usr/bin/observium_agent

	# configure the IPv[4\|6] address(es) of your Observium server here:
	# only_from = 127.0.0.1 ::1

	# Don't be too verbose. Don't log every check. This might be
	# commented out for debugging. If this option is commented out
	# the default options will be used for this service.
	log_on_success =

	disable = no
	}