Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Python logging configuration using JSON. If you want an updated example also covering YAML files, check here: https://github.com/pmav99/python-logging-example
{
"logging": {
"version": 1,
"disable_existing_loggers": true,
"formatters": {
"brief": {
"class": "logging.Formatter",
"datefmt": "%I:%M:%S",
"format": "%(levelname)-8s; %(name)-15s; %(message)s"
},
"single-line": {
"class": "logging.Formatter",
"datefmt": "%I:%M:%S",
"format": "%(levelname)-8s; %(asctime)s; %(name)-15s; %(module)s:%(funcName)s;%(lineno)d: %(message)s"
},
"multi-process": {
"class": "logging.Formatter",
"datefmt": "%I:%M:%S",
"format": "%(levelname)-8s; [%(process)d]; %(name)-15s; %(module)s:%(funcName)s;%(lineno)d: %(message)s"
},
"multi-thread": {
"class": "logging.Formatter",
"datefmt": "%I:%M:%S",
"format": "%(levelname)-8s; %(threadName)s; %(name)-15s; %(module)s:%(funcName)s;%(lineno)d: %(message)s"
},
"verbose": {
"class": "logging.Formatter",
"datefmt": "%I:%M:%S",
"format": "%(levelname)-8s; [%(process)d]; %(threadName)s; %(name)-15s; %(module)s:%(funcName)s;%(lineno)d: %(message)s"
},
"multiline": {
"class": "logging.Formatter",
"format": "Level: %(levelname)s\nTime: %(asctime)s\nProcess: %(process)d\nThread: %(threadName)s\nLogger: %(name)s\nPath: %(module)s:%(lineno)d\nFunction :%(funcName)s\nMessage: %(message)s\n"
}
},
"handlers": {
"console":{
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "single-line",
"stream" : "ext://sys.stdout"
},
"file_handler": {
"level": "INFO",
"class": "logging.handlers.WatchedFileHandler",
"formatter": "verbose",
"filename": "/tmp/file_handler.log",
"mode": "a",
"encoding": "utf-8"
},
"smtp": {
"level": "ERROR",
"class": "logging.handlers.SMTPHandler",
"formatter": "multiline",
"mailhost": ["127.0.0.1", 60025],
"fromaddr": "sender@example.com",
"toaddrs": ["recipient@example.com"],
"subject": "Something went wrong"
}
},
"loggers": { },
"root": {
"handlers": ["console", "smtp"],
"level": "DEBUG"
}
}
}
{
"logging": {
"version": 1,
"disable_existing_loggers": true,
"formatters": {
"brief": {
"class": "logging.Formatter",
"style": "{",
"datefmt": "%I:%M:%S",
"format": "{levelname:8s}; {name:<15s}; {message:s}"
},
"single-line": {
"class": "logging.Formatter",
"style": "{",
"datefmt": "%I:%M:%S",
"format": "{levelname:8s}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
},
"multi-process": {
"class": "logging.Formatter",
"style": "{",
"datefmt": "%I:%M:%S",
"format": "{levelname:8s}; {process:5d}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
},
"multi-thread": {
"class": "logging.Formatter",
"style": "{",
"datefmt": "%I:%M:%S",
"format": "{levelname:8s}; {threadName:5d}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
},
"verbose": {
"class": "logging.Formatter",
"style": "{",
"datefmt": "%I:%M:%S",
"format": "{levelname:8s}; {process:5d}; {threadName:8s}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
},
"multiline": {
"class": "logging.Formatter",
"style": "{",
"datefmt": "%I:%M:%S",
"format": "{levelname:8s}\n{process:5d}\n{threadName:8s}\n{asctime:s}\n{name:<15s}{lineno:4d}\n{message:s}\n"
}
},
"handlers": {
"console":{
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "single-line",
"stream" : "ext://sys.stdout"
},
"file_handler": {
"level": "INFO",
"class": "logging.handlers.WatchedFileHandler",
"formatter": "verbose",
"filename": "/tmp/file_handler.log",
"mode": "a",
"encoding": "utf-8"
},
"smtp": {
"level": "ERROR",
"class": "logging.handlers.SMTPHandler",
"formatter": "multiline",
"mailhost": ["127.0.0.1", 60025],
"fromaddr": "sender@example.com",
"toaddrs": ["recipient@example.com"],
"subject": "Something went wrong"
}
},
"loggers": { },
"root": {
"handlers": ["console", "smtp"],
"level": "DEBUG"
}
}
}
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" An example of using JSON for log configuration """
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import
import codecs
import json
import logging
import logging.config
import sys
def main():
# each time we need to log something we can create a logger object
# The operation of creating a logger should be quite cheap.
# getLogger() without arguments returns the "root" logger.
logger = logging.getLogger()
logger.info("This is an INFO message on the root logger.")
# If we need to separate things, we can always create child loggers:
child = logging.getLogger().getChild("child")
child.warning("This is a WARNING message on the child logger.")
# let's create an error. This will send an email
child.error("This is an ERROR message.")
if __name__ == "__main__":
# create an initial logger. It will only log to console and it will disabled
# when we read the logging configuration from the config file.
# This logger can be useful when we need early logging. E.g. we may want to log
# the location of the JSON file (e.g. if we get it from a CLI argument).
logging.basicConfig(level="INFO")
logger = logging.getLogger()
logger.info("This is the logger configured by `logging.basicConfig()`.")
# Load the configuration.
config_file = "config%d.json" % sys.version_info.major
with codecs.open(config_file, "r", encoding="utf-8") as fd:
config = json.load(fd)
# Set up proper logging. This one disables the previously configured loggers.
logging.config.dictConfig(config["logging"])
main()

Example for configuring logging in Python 2/3 using a JSON file.

If you run example.py as it is, then you will get the following output:

INFO:root:This is the logger configured by `logging.basicConfig()`.
INFO    ; 2014-10-18 15:17:38,278; root           ; This is an INFO message on the root logger.
WARNING ; 2014-10-18 15:17:38,278; child          ; This is a WARNING message on the child logger.
ERROR   ; 2014-10-18 15:17:38,278; child          ; This is an ERROR message.
--- Logging error ---
Traceback (most recent call last):
  File "/usr/lib/python3.4/logging/handlers.py", line 971, in emit
    smtp = smtplib.SMTP(self.mailhost, port, timeout=self.timeout)
  File "/usr/lib/python3.4/smtplib.py", line 242, in __init__
    (code, msg) = self.connect(host, port)
  File "/usr/lib/python3.4/smtplib.py", line 321, in connect
    self.sock = self._get_socket(host, port, self.timeout)
  File "/usr/lib/python3.4/smtplib.py", line 292, in _get_socket
    self.source_address)
  File "/usr/lib/python3.4/socket.py", line 509, in create_connection
    raise err
  File "/usr/lib/python3.4/socket.py", line 500, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
Call stack:
  File "foo.py", line 44, in <module>
    main()
  File "foo.py", line 29, in main
    child.error("This is an ERROR message.")
Message: 'This is an ERROR message.'
Arguments: ()

This traceback is not a related to our application. It has to do with the logging configuration. More specifically we have set an smtp handler that tries to send an email using a MailServer that runs on 127.0.0.1:60025, but there is no server listening to that ip/port. While developing, we can easily setup such a server by running the following command on a separate console session:

python3 -m smtpd -n -c DebuggingServer localhost:60025 

Needless to say, on production we should set up a proper MailServer. Anyway, now, if we execute the script we will get the following output:

INFO:root:This is the logger configured by `logging.basicConfig()`.
INFO    ; 2014-10-18 15:17:51,078; root           ; This is an INFO message on the root logger.
WARNING ; 2014-10-18 15:17:51,079; child          ; This is a WARNING message on the child logger.
ERROR   ; 2014-10-18 15:17:51,079; child          ; This is an ERROR message.

while, on the console that runs the Mail Server we will get the following output:

---------- MESSAGE FOLLOWS ----------
From: sender@example.com
To: recipient@example.com
Subject: Something went wrong
Date: Sat, 18 Oct 2014 12:17:51 -0000
X-Peer: 127.0.0.1

thread: MainThread
Level: ERROR
Time: 2014-10-18 15:17:51,079
Location: foo.py:29
Method: main
Message:
This is an ERROR message.
------------ END MESSAGE ------------
@ghost

This comment has been minimized.

Copy link

@ghost ghost commented Jun 18, 2018

This is the best guide. Thank you so much. I have a problem. I wanted to write logs in json file format how can I change my config file to dump data into json format. Any help would be appreciated.

@feechka1976

This comment has been minimized.

Copy link

@feechka1976 feechka1976 commented Jan 17, 2019

Hi! fist of the all - thanks for this line
"stream" : "ext://sys.stdout"
I spent hours to understand why I have error
File "/usr/local/lib/python2.7/logging/init.py", line 889, in emit
stream.write(fs % msg)
AttributeError: 'str' object has no attribute 'write'

my line was
"stream" : "sys.stdout"

and your line solved this issue!!

but I found something in your example, that didn't work for me:
"loggers": { },
"root": {
"handlers": ["console"],
"level": "DEBUG"
}

I got error: No handlers could be found for logger "root"

only when I set it is this way - it worked

"loggers": {
"root": {
"handlers": ["console"],
"level": "DEBUG"
}

thanks again!!!!!
},

@vkroz

This comment has been minimized.

Copy link

@vkroz vkroz commented Jan 28, 2019

There is an error in json example logging2.json, entry "logging": shouldn't be there. Logger config expects config object of the format

{
        "version": 1,
        "disable_existing_loggers": true,
        "formatters": {
        . . .
 }
@pmav99

This comment has been minimized.

Copy link
Owner Author

@pmav99 pmav99 commented Feb 9, 2019

@ghost What you want is called "structured logging". you might want to have a look here and at structlog

@vkroz Yeap, you are right, at some revision I must have done a copy paste error or something. I've fixed it. I've also added a link to a repo with yaml config if you are interested.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment