Skip to content

Instantly share code, notes, and snippets.

@rmg rmg/elasticsearch.log
Last active Aug 29, 2015

Embed
What would you like to do?
Log Aggregation
{"message":"ARGV: [\"/usr/local/bin/node\",\".\"]","@version":"1","@timestamp":"2014-06-20T16:13:29.117Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"DEBUG","tags":["DEBUG"]}
{"message":"application started","@version":"1","@timestamp":"2014-06-20T16:13:29.118Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"INFO","tags":["INFO"]}
{"message":"about to do something horrible!","@version":"1","@timestamp":"2014-06-20T16:13:29.118Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"WARN","tags":["WARN"]}
{"message":"[Error: Something horrible!]","@version":"1","@timestamp":"2014-06-20T16:13:29.118Z","host":"Ryans-StrongMac.local","path":"/var/log/strong-supervisor/logger-app.log","pid":"89371","worker":"1","level":"ERROR","tags":["ERROR"]}
var logger = require('./logger');
logger.debug('ARGV: %j', process.argv);
logger.info('application started');
logger.warn('about to do something horrible!');
logger.error(new Error('Something horrible!'));
var util = require('util');
var levels = [
'DEBUG', 'INFO', 'WARN', 'ERROR'
];
levels.forEach(function(level) {
module.exports[level.toLowerCase()] = function() {
console.log('%s %s', level, util.format.apply(util, arguments));
};
});
/var/log/my-app.log {
daily
compress
postrotate
kill -SIGUSR2 `cat /var/run/my-app.pid` > /dev/null
endscript
}
input {
file {
path => ["/var/log/my-app.log"]
}
}
filter {
grok {
match => [
# Timestamped
"message", "%{TIMESTAMP_ISO8601:extracted_timestamp} pid:%{INT:pid} worker:%{WORD:worker} %{LOGLEVEL:level} %{GREEDYDATA:message}",
# Timestamped but no log level
"message", "%{TIMESTAMP_ISO8601:extracted_timestamp} pid:%{INT:pid} worker:%{WORD:worker} %{GREEDYDATA:message}",
# Fallback match lines without timestamps
"message", "pid:%{INT:pid} worker:%{WORD:worker} %{LOGLEVEL:level} %{GREEDYDATA:message}",
"message", "pid:%{INT:pid} worker:%{WORD:worker} %{GREEDYDATA:message}"
]
overwrite => [ "message" ]
}
date {
# If a timestamp was extracted, use it for the event's timestamp
match => [ "extracted_timestamp", "ISO8601" ]
remove_field => [ "extracted_timestamp" ]
}
if ! [level] {
mutate { add_field => { level => "INFO" } }
}
mutate { add_tag => [ "%{level}" ] }
}
output {
elasticsearch {
host => "localhost"
protocol => "http"
}
}
2014-06-20T15:26:37.907Z pid:89028 worker:1 DEBUG ARGV: ["/usr/local/bin/node","."]
2014-06-20T15:26:37.908Z pid:89028 worker:1 INFO application started
2014-06-20T15:26:37.908Z pid:89028 worker:1 WARN about to do something horrible!
2014-06-20T15:26:37.908Z pid:89028 worker:1 ERROR [Error: Something horrible!]
DEBUG ARGV: ["node","/Users/ryan/work/strong-supervisor/test/logger-app"]
INFO application started
WARN about to do something horrible!
ERROR [Error: Something horrible!]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.