Log Aggregation

elasticsearch.log

{"message":"ARGV: [\"/usr/local/bin/node\",\".\"]","@version":"1","@timestamp":"2014-06-20T16:13:29.117Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"DEBUG","tags":["DEBUG"]}
{"message":"application started","@version":"1","@timestamp":"2014-06-20T16:13:29.118Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"INFO","tags":["INFO"]}
{"message":"about to do something horrible!","@version":"1","@timestamp":"2014-06-20T16:13:29.118Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"WARN","tags":["WARN"]}
{"message":"[Error: Something horrible!]","@version":"1","@timestamp":"2014-06-20T16:13:29.118Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"ERROR","tags":["ERROR"]}

index.js

var logger = require('./logger');

logger.debug('ARGV: %j', process.argv);
logger.info('application started');
logger.warn('about to do something horrible!');
logger.error(new Error('Something horrible!'));

logger.js

var util = require('util');
var levels = [
  'DEBUG', 'INFO', 'WARN', 'ERROR'
];
levels.forEach(function(level) {
  module.exports[level.toLowerCase()] = function() {
    console.log('%s %s', level, util.format.apply(util, arguments));
  };
});

logrotate.conf

/var/log/my-app.log {
  daily
  compress
  postrotate
    kill -SIGUSR2 `cat /var/run/my-app.pid` > /dev/null
  endscript
}

logstash.conf

input {
    file {
        path => ["/var/log/my-app.log"]
    }
}

filter {
    grok {
        match => [
            # Timestamped
            "message", "%{TIMESTAMP_ISO8601:extracted_timestamp} pid:%{INT:pid} worker:%{WORD:worker} %{LOGLEVEL:level} %{GREEDYDATA:message}",
            # Timestamped but no log level
            "message", "%{TIMESTAMP_ISO8601:extracted_timestamp} pid:%{INT:pid} worker:%{WORD:worker} %{GREEDYDATA:message}",
            # Fallback match lines without timestamps
            "message", "pid:%{INT:pid} worker:%{WORD:worker} %{LOGLEVEL:level} %{GREEDYDATA:message}",
            "message", "pid:%{INT:pid} worker:%{WORD:worker} %{GREEDYDATA:message}"
            ]
        overwrite => [ "message" ]
    }
    date {
        # If a timestamp was extracted, use it for the event's timestamp
        match => [ "extracted_timestamp", "ISO8601" ]
        remove_field => [ "extracted_timestamp" ]
    }
    if ! [level] {
        mutate { add_field => { level => "INFO" } }
    }
    mutate { add_tag => [ "%{level}" ] }
}

output {
    elasticsearch {
        host => "localhost"
        protocol => "http"
    }
}

stdout.log

2014-06-20T15:26:37.907Z pid:89028 worker:1 DEBUG ARGV: ["/usr/local/bin/node","."]
2014-06-20T15:26:37.908Z pid:89028 worker:1 INFO application started
2014-06-20T15:26:37.908Z pid:89028 worker:1 WARN about to do something horrible!
2014-06-20T15:26:37.908Z pid:89028 worker:1 ERROR [Error: Something horrible!]

stdout.txt

DEBUG ARGV: ["node","/Users/ryan/work/strong-supervisor/test/logger-app"]
INFO application started
WARN about to do something horrible!
ERROR [Error: Something horrible!]