Skip to content

Instantly share code, notes, and snippets.

@drillbits
Created June 26, 2018 02:41
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save drillbits/78ffd49e946b3e859a25b4da89134ec9 to your computer and use it in GitHub Desktop.
Save drillbits/78ffd49e946b3e859a25b4da89134ec9 to your computer and use it in GitHub Desktop.
Python, syslog, fluentd
from logging import getLogger, StreamHandler, Formatter
from logging.handlers import SysLogHandler
import logging
from socket import gethostname
logger = getLogger(__name__)
logger.setLevel(logging.DEBUG)
syslog_fmt = Formatter(
fmt='%(asctime)s %(hostname)s %(name)s[%(process)s]: %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S%z')
local_syslog_handler = SysLogHandler(address='/var/run/syslog')
# default format: b'<10>this is critical\x00'
# in system.log: Jun 26 10:30:51 bebe Unknown: this is critical
remote_syslog_handler = SysLogHandler(address=('0.0.0.0', 514))
# default format: b'<10>this is critical\x00'
# in fluentd: "<10>this is critical\x00"
# text to parse: "this is critical\x00"
remote_syslog_handler.formatter = syslog_fmt
# syslog_fmt: b'<10>2018-06-26T11:25:52+0900 bebe.local __main__[6353]: this is critical\x00'
# in fluentd: "<10>2018-06-26T11:25:52+0900 bebe.local __main__[6353]: this is critical\x00"
# text to parse: "2018-06-26T11:25:52+0900 bebe.local __main__[6353]: this is critical\x00"
#
# => {
# "time"=>"2018-06-26T11:25:52+0900",
# "host"=>"bebe.local",
# "ident"=>"__main__",
# "pid"=>"6353",
# "message"=>"this is critical\x00"
# }
stream_handler = StreamHandler()
logger.addHandler(local_syslog_handler)
logger.addHandler(remote_syslog_handler)
logger.addHandler(stream_handler)
logger.critical('this is critical', extra={'hostname': gethostname()})
<source>
@type syslog
port 514
bind 0.0.0.0
tag syslog.test
<parse>
@type regexp
expression /^(?<time>[^ ]+) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
time_key time
time_format %Y-%m-%dT%H:%M:%S%z
</parse>
</source>
<match syslog.**>
@type file
path /var/log/test
time_slice_format %Y%m%d%H%M
time_slice_wait 1m
</match>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment