ajohnstone

mysql> SELECT * FROM icinga_conninfo;
+-------------+-------------+------------------------+---------------+-------------+----------------+--------------+---------------------+---------------------+---------------------+---------------------+---------------------+-----------------+-----------------+-------------------+
| conninfo_id | instance_id | agent_name             | agent_version | disposition | connect_source | connect_type | connect_time        | disconnect_time     | last_checkin_time   | data_start_time     | data_end_time       | bytes_processed | lines_processed | entries_processed |
+-------------+-------------+------------------------+---------------+-------------+----------------+--------------+---------------------+---------------------+---------------------+---------------------+---------------------+-----------------+-----------------+-------------------+
|           1 |           1 | IDOMOD                 | 1.7.1         | REALTIME    | TCPSOCKET      | INITIAL      | 2012-11-09 16:03:47 

ajohnstone

icinga.cfg

# Uncomment this to enable idomod.o
# Don't forget to also enable the daemon in /etc/default/icinga
broker_module=/usr/lib/icinga/idomod.so config_file=/etc/icinga/idomod.cfg

in modules/idomod.cfg i have
define module {
       module_name     ido_mod
       path            /usr/lib/icinga/idomod.so

ajohnstone

root@ip-10-239-75-244:~# FACTER_server_tags="role:hadoop-master=true,role:hadoop-worker=true" puppet apply --verbose -vvv --debug --modulepath /etc/puppet/modules /etc/puppet/manifests/site.pp 
Debug: importing '/etc/puppet/modules/truth/manifests/enforcer.pp' in environment production
Debug: Automatically imported truth::enforcer from truth/enforcer into production
Error: Unknown function has_role at /etc/puppet/modules/truth/manifests/enforcer.pp:4 on node ip-10-239-75-244.eu-west-1.compute.internal
Error: Unknown function has_role at /etc/puppet/modules/truth/manifests/enforcer.pp:4 on node ip-10-239-75-244.eu-west-1.compute.internal

ajohnstone

# Uncomment the next line to enable TCP/IP SYN cookies
net.ipv4.tcp_syncookies=0

# Uncomment the next two lines to enable Spoof protection (reverse-path filter)
# Turn on Source Address Verification in all interfaces to
# prevent some spoofing attacks
net.ipv4.conf.default.rp_filter=1

net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

ajohnstone

mkdir -p /opt/graphite/storage
mount -onoatime,nodiratime,nobarrier -t xfs /dev/xvdf  /opt/graphite/storage

find /opt/graphite/storage/whisper/stats/collectd_bk -type f | while read file; do
  dest=$(echo $file | sed 's/collectd_bk/collectd/g')
  mkdir -p $(dirname $dest)
  echo $dest;
  rsync --recursive --perms --times --whole-file --ignore-existing $file $dest;
  if [ -e "$file" ]; then

ajohnstone

[cache]
# Configure carbon directories.
#
# OS environment variables can be used to tell carbon where graphite is
# installed, where to read configuration from and where to write data.
#
#   GRAPHITE_ROOT        - Root directory of the graphite installation.
#                          Defaults to ../
#   GRAPHITE_CONF_DIR    - Configuration directory (where this file lives).
#                          Defaults to $GRAPHITE_ROOT/conf/

ajohnstone

Carbon relay - performance
alias(color(sumSeries(group(carbon.agents.*.updateOperations)), "blue"),"Updates") 
alias(color(sumSeries(group(carbon.agents.*.metricsReceived)), "green"), "Metrics Received")
alias(color(sumSeries(group(carbon.agents.*.committedPoints)),"orange"),"Committed Points"))
alias(secondYAxis(color(sumSeries(group(carbon.agents.*.pointsPerUpdate)),"yellow")),"PPU")
alias(secondYAxis(color(averageSeries(group(carbon.agents.*.cpuUsage)),"red")),"CPU (avg)")
alias(secondYAxis(color(sumSeries(group(carbon.agents.*.creates)),"purple")),"Creates")

Metrics received
carbon.agents.*.metricsReceived

ajohnstone

root      1475 33.5  0.5 467484 362016 ?       Sl   14:55 122:38 /usr/bin/python /opt/graphite/bin/carbon-cache.py --config /opt/graphite/conf/carbon-cache.conf --instance=c start
root      1483 39.6  0.9 727544 636780 ?       Sl   14:55 144:36 /usr/bin/python /opt/graphite/bin/carbon-cache.py --config /opt/graphite/conf/carbon-cache.conf --instance=b start
root      1491 25.5 28.2 19938972 19854664 ?   Sl   14:55  93:12 /usr/bin/python /opt/graphite/bin/carbon-cache.py --config /opt/graphite/conf/carbon-cache.conf --instance=a start

ajohnstone

LoadPlugin "apache"
<Plugin "apache">
   <Instance "apache80">
       URL "http://localhost/server-status?auto"
   </Instance>

   <Instance "apache81">
       URL "http://localhost:81/server-status?auto"
   </Instance>
</Plugin>

ajohnstone

zcat Summary.2012-12-09.gz | grep '2012-12-09 20:[234]' | awk '
  BEGIN{
    size=50;
    tmax = -999999999;
    tmin = 999999999;
  }
  {   # Accumulate basic data
      cnt[$10]++;
      item[++n] = $10;
      if ($10 > max) max = $10; if ($10 < min) min = $10;