TOML/JSON logging config for dowwie

config.toml

[logging.main]
version = 1

[logging.main.formatters.generic]
"()" = "formatter.JSONFormatter"

[logging.main.handlers.console]
formatter = "generic"
class = "logging.StreamHandler"
stream = "ext://sys.stdout"
level = "NOTSET"

[logging.main.root]
level = "INFO"
handlers = ["console"]

[logging.main.loggers.sample]
level = "DEBUG"
handlers = []

formatter.py

import json
import logging
import datetime
import traceback as tb
import itertools


class JSONFormatter(logging.Formatter):

    def __init__(self,
                 fmt=None,
                 datefmt=None,
                 json_cls=None,
                 json_default=None):
        """
        :param fmt: Config as a JSON string, allowed fields;
               source_host: override source host name
        :param datefmt: Date format to use (required by logging.Formatter
            interface but not used)
        :param json_cls: JSON encoder to forward to json.dumps
        :param json_default: Default JSON representation for unknown types,
                             by default coerce everything to a string
        """

        if fmt is not None:
            self._fmt = json.loads(fmt)
        else:
            self._fmt = {}

        self.json_default = json_default if json_default\
            else self._default_json_default()

        self.json_cls = json_cls
        if 'source_host' in self._fmt:
            self.source_host = self._fmt['source_host']
        else:
            try:
                self.source_host = socket.gethostname()
            except:
                self.source_host = ""

    def _default_json_default(obj):
        """
        Coerce everything to strings.
        All objects representing time get output as ISO8601.
        """
        if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
            return obj.isoformat()
        else:
            return str(obj)
    
    def format_exception(self, ei, strip_newlines=True):
        lines = tb.format_exception(*ei)
        if strip_newlines:
            lines = [(line.rstrip().splitlines()) for line in lines]
            lines = list(itertools.chain(*lines))
        return lines

    def format(self, record):
        """
        Format a log record to JSON, if the message is a dict
        assume an empty message and use the dict as additional
        fields.
        """

        fields = record.__dict__.copy()

        if isinstance(record.msg, dict):
            fields.update(record.msg)
            fields.pop('msg')
            msg = ""
        else:
            msg = record.getMessage()

        if 'msg' in fields:
            fields.pop('msg')

        if 'exc_info' in fields:
            if fields['exc_info']:
                formatted = self.format_exception(fields['exc_info'])
                fields['exception'] = formatted
            fields.pop('exc_info')

        if 'exc_text' in fields and not fields['exc_text']:
            fields.pop('exc_text')

        logr = {} 

        logr.update({'@message': msg,
                     '@timestamp': datetime.datetime.utcnow().
                     strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
                     '@source_host': self.source_host,
                     '@fields': self._build_fields(logr, fields)})

        return json.dumps(logr, default=self.json_default, cls=self.json_cls)

    def _build_fields(self, defaults, fields):
        """Return provided fields including any in defaults

        >>> f = JSONFormatter()
        # Verify that ``fields`` is used
        >>> f._build_fields({}, {'foo': 'one'}) == \
                {'foo': 'one'}
        True
        # Verify that ``@fields`` in ``defaults`` is used
        >>> f._build_fields({'@fields': {'bar': 'two'}}, {'foo': 'one'}) == \
                {'foo': 'one', 'bar': 'two'}
        True
        # Verify that ``fields`` takes precedence
        >>> f._build_fields({'@fields': {'foo': 'two'}}, {'foo': 'one'}) == \
                {'foo': 'one'}
        True
        """
        c = {}
        c.update(defaults.get('@fields', {}))
        c.update(fields.items())
        return c

output.json

{"@fields": {"threadName": "MainThread", "name": "sample", "thread": 140735108260608, "relativeCreated": 62.41297721862793, "process": 4362, "args": [], "module": "testit", "funcName": "log_some_stuff", "levelno": 10, "processName": "MainProcess", "created": 1434483785.980757, "msecs": 980.7569980621338, "pathname": "testit.py", "filename": "testit.py", "levelname": "DEBUG", "lineno": 9}, "@timestamp": "2015-06-16T19:43:05.980816Z", "@source_host": "", "@message": "fnord"}
{"@fields": {"threadName": "MainThread", "name": "sample", "thread": 140735108260608, "relativeCreated": 62.96396255493164, "process": 4362, "args": [], "module": "testit", "funcName": "log_some_stuff", "levelno": 20, "processName": "MainProcess", "created": 1434483785.981308, "msecs": 981.3079833984375, "pathname": "testit.py", "filename": "testit.py", "levelname": "INFO", "lineno": 10}, "@timestamp": "2015-06-16T19:43:05.981343Z", "@source_host": "", "@message": "foobar"}
{"@fields": {"threadName": "MainThread", "name": "sample", "thread": 140735108260608, "relativeCreated": 63.09390068054199, "process": 4362, "args": [], "module": "testit", "funcName": "log_some_stuff", "levelno": 40, "processName": "MainProcess", "created": 1434483785.981438, "msecs": 981.4379215240479, "pathname": "testit.py", "filename": "testit.py", "levelname": "ERROR", "lineno": 11}, "@timestamp": "2015-06-16T19:43:05.981463Z", "@source_host": "", "@message": "hello"}
{"@fields": {"exception": ["Traceback (most recent call last):", "  File \"testit.py\", line 13, in log_some_stuff", "    {}['fnord']", "KeyError: 'fnord'"], "threadName": "MainThread", "name": "sample", "thread": 140735108260608, "relativeCreated": 63.20309638977051, "process": 4362, "args": [], "module": "testit", "funcName": "log_some_stuff", "levelno": 40, "processName": "MainProcess", "created": 1434483785.981547, "msecs": 981.5471172332764, "pathname": "testit.py", "filename": "testit.py", "levelname": "ERROR", "lineno": 15}, "@timestamp": "2015-06-16T19:43:05.981676Z", "@source_host": "", "@message": "an exception"}

test_logging.py

from montague.loadwsgi import Loader
import os
import logging
import logging.config

log = logging.getLogger("sample")

def log_some_stuff():
	log.debug("fnord")
	log.info("foobar")
	log.error("hello")
	try:
		{}['fnord']
	except:
		log.exception("an exception")

def setup():
	path = os.path.join(os.getcwd(), 'config.toml')
	loader = Loader(path)
	logging_config = loader.logging_config()
	logging.config.dictConfig(logging_config)

if __name__ == '__main__':
	setup()
	log_some_stuff()