Semantic Logging

Downloading the sample

git clone git://github.com/haf/semantic-logging.git
cd semantic-logging
git submodule update --init
./download.sh
vagrant up

gistfile1.cs

using NLog;

namespace ConsoleApplication13
{
	class Program
	{
		static readonly Logger _logger = LogManager.GetCurrentClassLogger();

		static void Main(string[] args)
		{
			_logger.Debug("Hello World from NLog!");

			_logger.InfoTag("user clicked purchase", "shop-actions");

			Shutdown();
		}

		static void Shutdown()
		{
			var allTargets = LogManager.Configuration.AllTargets;

			foreach (var target in allTargets)
				target.Dispose();
		}
	}
}

kibana.pp

class kibana {

  $kpath = '/tmp/kibana.tar.gz'
  $wwwpath = '/var/www/kibana'

  file { "$kpath":
    source => 'puppet:///files/rashidkpc-Kibana-v0.1.5-44-gf5c80c3.tar.gz',
    ensure => present,
    mode => '0755'
  }
  
  file { '/var/www':
  	ensure => 'directory',
  	mode => '0755',
  	owner => 'root',
  	group => 'root'
  }
  
  exec { "$kpath":
    command => "tar -zxvf $kpath -C /var/www && mv `ls /var/www | head -n 1` $wwwpath",
    unless  => "find $wwwpath",
    creates => "$wwwpath",
    require => [
      File['/var/www'],
      File["$kpath"]
    ],
    path    => "/usr/local/bin:/usr/sbin:/usr/bin:/bin",
    cwd     => '/var/www'
  }

  package { 'php':
    name => ["php5-cli", "php5-common", "php5-suhosin", "php5-curl"],
    ensure => present
  }

  package { 'php-fpm':
    name => [ "php5-fpm", "php5-cgi" ],
    ensure => present,
    require => Package['php']
  }

  file { "$wwwpath":
    ensure => "directory",
    owner => 'kibana',
    group => 'kibana',
    mode => '0755',
    require => Exec["$kpath"]
  }

  group { 'kibana':
    ensure => present
  }
  
  user { 'kibana':
    ensure => present,
    gid => 'kibana',
    home => $wwwpath,
    managehome => true,
    require => Group['kibana'] 
  }
  
  service { 'php5-fpm':
  	ensure => running,
  	enable => true,
  	hasrestart => true,
  	hasstatus => true,
  	require => Package['php-fpm']
  }
  
  host { 'elasticsearch':
  	ip => '::1',
  }

  include nginx
  include nginx::fcgi
  
  nginx::site { 'default':
    ensure => absent
  }

  nginx::fcgi::site { 'nginx-kibana-enable':
    root            => "$wwwpath",
    fastcgi_pass    => "127.0.0.1:9000",
    server_name     => ["localhost", "$::hostname", "$::fqdn"],
    require         => [ 
      Service['php5-fpm'] ,
      Host['elasticsearch']
    ],
    template        => 'nginx/fcgi_site.erb',
    listen          => '80'
  }
}

LogStash upstart

description     "LogStash + Web"

start on (net-device-up and local-filesystems and runlevel [2345])
stop on runlevel [016]
respawn
respawn limit 10 5

console output

exec /usr/bin/java  -jar /opt/logstash-1.1.1-monolithic.jar agent -f /etc/logstash.conf --log /var/log/logstash.log -- web

Program.cs

// Copyright 2012 Henrik Feldt

using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Threading;
using NLog;
using System.Linq;

namespace insert_data
{
	internal class Program
	{
		// will feed some message lines to the 
		static void Main()
		{
			var logger = LogManager.GetCurrentClassLogger();
			var now = DateTime.UtcNow;
			var from = now.Subtract(TimeSpan.FromMinutes(15));

			GenerateData(logger, now, @from);
		}

		static void GenerateData(Logger logger, DateTime init, DateTime @from)
		{
			var rand = new Random();
			var levels = new[] { LogLevel.Trace, LogLevel.Debug, LogLevel.Info, LogLevel.Warn };
			var messages = File.ReadAllLines(GetPath("messages.txt"));
			var tags = File.ReadAllText(GetPath("tags.txt")).Split(' ').Select(t => t.Trim(new[] { '.' })).ToArray();

			Console.WriteLine("Starting live data generation.");

			while (true)
			{
				logger.Log(levels[rand.Next(levels.Length)],
					messages[rand.Next(messages.Length)],
					tags: new[] { tags[rand.Next(tags.Length)] },
					fields: new Dictionary<string, object>
						{
							{ "curr-position", DateTime.UtcNow - init }
						});

				Thread.Sleep(20);
			}
		}

		static string GetPath(string filename)
		{
			var codeBase = new Uri(Assembly.GetEntryAssembly().GetName().CodeBase).AbsolutePath;
			var codeDir = Path.GetDirectoryName(codeBase);
			var filePath = Path.Combine(codeDir, filename);
			return filePath;
		}
	}
}

semantic-logging.pp

node 'semantic-logging' {
  class { 'aptupdate':
    stage => pre
  }
  
  include rabbitmq
  rabbitmq::vhost { '/': }
  rabbitmq::plugin { 'rabbitmq_management': }~>
  class { 'elasticsearch':
    version      => '0.19.8',
    java_package => 'openjdk-6-jre-headless',
    dbdir        => '/var/lib/elasticsearch',
    logdir       => '/var/log/elasticsearch',
  }~>  
  class { 'logstash': }~>
  class { 'kibana': }
}

server.pp

  package { $rabbitmq::package:
    ensure  => installed,
    require => Exec['apt-get update rabbit'],
  }

  service { $rabbitmq::service:
    ensure  => running,
    enable  => true,
    require => Package[$rabbitmq::package],
  }

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant::Config.run do |config|
  config.vm.host_name = 'semantic-logging'
  config.vm.box = "precise32"
  config.vm.box_url = "http://files.vagrantup.com/precise32.box"
  config.vm.forward_port 9292, 9292 # logstash default web
  config.vm.forward_port 5672, 5672 # rabbitmq
  config.vm.forward_port 9200, 9200 # elasticsearch
  config.vm.forward_port 80, 8080 # kibana
  config.vm.forward_port 55672, 55672 # rabbitmq management

  config.vm.share_folder "puppet-files", "/etc/puppet/files", "./files"
  
  config.vm.customize do |vm|
    vm.memory_size = 1024
  end

  config.vm.provision :puppet do |puppet|
    puppet.module_path = "./modules"
    puppet.manifests_path = "."
    puppet.manifest_file = "manifests/semantic-logging.pp"
    puppet.options = ["--fileserverconfig=/vagrant/fileserver.conf", "--verbose", "--debug" ]
  end
end