Setting up Logging in Python and Flask Application the right way

app.py

import json
import requests
import logging
import os
from logging.config import dictConfig

# debug settings
debug = eval(os.environ.get("DEBUG", "False"))

from flask import Flask, make_response, request

# for sending error logs to slack
class HTTPSlackHandler(logging.Handler):
    def emit(self, record):
        log_entry = self.format(record)
        json_text = json.dumps({"text": log_entry})
        url = 'https://hooks.slack.com/services/<org_id>/<api_key>'
        return requests.post(url, json_text, headers={"Content-type": "application/json"}).content


dictConfig({
    "version": 1,
    "disable_existing_loggers": True,
    "formatters": {
        "default": {
            "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
        },
        "access": {
            "format": "%(message)s",
        }
    },
    "handlers": {
        "console": {
            "level": "INFO",
            "class": "logging.StreamHandler",
            "formatter": "default",
            "stream": "ext://sys.stdout",
        },
        "email": {
            "class": "logging.handlers.SMTPHandler",
            "formatter": "default",
            "level": "ERROR",
            "mailhost": ("smtp.example.com", 587),
            "fromaddr": "devops@example.com",
            "toaddrs": ["receiver@example.com", "receiver2@example.com"],
            "subject": "Error Logs",
            "credentials": ("username", "password"),
        },
        "slack": {
            "class": "app.HTTPSlackHandler",
            "formatter": "default",
            "level": "ERROR",
        },
        "error_file": {
            "class": "logging.handlers.RotatingFileHandler",
            "formatter": "default",
            "filename": "/var/log/gunicorn.error.log",
            "maxBytes": 10000,
            "backupCount": 10,
            "delay": "True",
        },
        "access_file": {
            "class": "logging.handlers.RotatingFileHandler",
            "formatter": "access",
            "filename": "/var/log/gunicorn.access.log",
            "maxBytes": 10000,
            "backupCount": 10,
            "delay": "True",
        }
    },
    "loggers": {
        "gunicorn.error": {
            "handlers": ["console"] if debug else ["console", "slack", "error_file"],
            "level": "INFO",
            "propagate": False,
        },
        "gunicorn.access": {
            "handlers": ["console"] if debug else ["console", "access_file"],
            "level": "INFO",
            "propagate": False,
        }
    },
    "root": {
        "level": "DEBUG" if debug else "INFO",
        "handlers": ["console"] if debug else ["console", "slack"],
    }
})


app = Flask(__name__)


@app.route("/status", methods=["GET"])
def health_check():
    logging.debug("debug log")
    logging.info("info log")
    logging.warning("warning log")
    logging.error("error log")
    # logging.exception("exception log")
    
    return make_response("OK", 200)


if __name__ == "__main__":
    app.run(debug=debug, host="0.0.0.0", port="5000")

# Run with Gunicorn
# gunicorn app:app -b 0.0.0.0:5000 --workers 2 -k gevent --timeout 300 --worker-connections 1000 --max-requests 1000000 --limit-request-line 8190 --access-logfile '-' --error-logfile '-'

requirements.txt

Flask==1.1.2
requests==2.23.0
gunicorn[gevent]==20.0.4

Hello .
What will be the setup in order to log messages to two different TimeRotating files ( each file will log messages from different "functional modules" ). I defined the following loggers.py file :

import logging

from logging import FileHandler
from logging import Formatter
from logging.handlers import TimedRotatingFileHandler
from defaultapp.config import Config

import os


LOG_FORMAT = ("%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d")

if(Config.LOG_LEVEL=='INFO'):
   LOG_LEVEL = logging.INFO
elif(Config.LOG_LEVEL=='DEBUG'):
   LOG_LEVEL = logging.DEBUG
elif(Config.LOG_LEVEL=='ERROR'):
   LOG_LEVEL = logging.ERROR
else:
   LOG_LEVEL = logging.INFO


root = os.path.dirname(os.path.abspath(__file__))
logdir = os.path.join(root, 'logs')
if not os.path.exists(logdir):
   os.mkdir(logdir)


# diarios logger
diarios_LOG_FILE = os.path.join(logdir,"diarios.log")


diarios_logger = logging.getLogger("diarios")
diarios_logger.setLevel(LOG_LEVEL)

# diarios_handler = TimedRotatingFileHandler(diarios_LOG_FILE,
#                                        when="m",
#                                        interval=1,
#                                        backupCount=5,
#                                        encoding='utf-8')

diarios_handler = FileHandler(diarios_LOG_FILE,encoding='utf-8')
diarios_handler.setLevel(LOG_LEVEL)
diarios_handler.setFormatter(Formatter(LOG_FORMAT))

diarios_logger.addHandler(diarios_handler)

# portafolio logger
patrimonio_LOG_FILE = os.path.join(logdir,"patrimonio.log")
patrimonio_logger = logging.getLogger("patrimonio")

patrimonio_logger.setLevel(LOG_LEVEL)
patrimonio_handler = FileHandler(patrimonio_LOG_FILE,encoding='utf-8')
# patrimonio_handler = TimedRotatingFileHandler(patrimonio_LOG_FILE,
#                                        when="W0",
#                                        interval=1,
#                                        backupCount=5,
#                                        encoding='utf-8')
patrimonio_handler.setLevel(LOG_LEVEL)
patrimonio_handler.setFormatter(Formatter(LOG_FORMAT))
patrimonio_logger.addHandler(patrimonio_handler)


# common logger
common_LOG_FILE = os.path.join(logdir,"common.log")
common_logger = logging.getLogger("common")

common_logger.setLevel(LOG_LEVEL)
common_handler = FileHandler(common_LOG_FILE,encoding='utf-8')
# common_handler = TimedRotatingFileHandler(common_LOG_FILE,
#                                        when="W0",
#                                        interval=1,
#                                        backupCount=5,
#                                        encoding='utf-8')
common_handler.setLevel(LOG_LEVEL)
common_handler.setFormatter(Formatter(LOG_FORMAT))
common_logger.addHandler(common_handler)

and import either diarios_logger, or patrimonio_logger or common_logger in other modules where I want to log messages to , but got the following error:

PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:

I have searched this error message in the web and found the reason is because flask-wsgi is runs multiprocessing workers and they compete to handle and rotate the log file , creating a mess!!
I also found one possible solution to this concurrency lock issue is to use QueueHandlers and QueueListeners...!

Thanks in advance for any hint or advice you could provide.

Rgds

Hello

I finally solved the problem I got .

I defined my own dictConfig :

import logging
from defaultapp.config import Config
from logging.config import dictConfig
import os


root = os.path.dirname(os.path.abspath(__file__))
logdir = os.path.join(root, 'logs')
if not os.path.exists(logdir):
   os.mkdir(logdir)

# debug settings
debug = Config.DEBUG


dictConfig({
    "version": 1,
    "disable_existing_loggers": True,
    "formatters": {
        "default": {
            "format": "[%(asctime)s] %(levelname)s in %(module)s - %(lineno)d: %(message)s ",
        }
    },
    "handlers": {


    	"console": {
            "level": Config.LOG_LEVEL,
            "class": "logging.StreamHandler",
            "formatter": "default",
            "stream": "ext://flask.logging.wsgi_errors_stream",
        },
        
        "diarios_file": {
            "level": Config.LOG_LEVEL,
            "class": "logging.handlers.TimedRotatingFileHandler",
            "formatter": "default",
            "filename": os.path.join(logdir,"diarios.log"),
            "when": "m",
            "interval": 1,
            "backupCount": 5,
            "encoding":"utf8"
        },

       "patrimonio_file": {
            "level": Config.LOG_LEVEL,
            "class": "logging.handlers.TimedRotatingFileHandler",
            "formatter": "default",
            "filename": os.path.join(logdir,"patrimonio.log"),
            "when": "m",
            "interval": 1,
            "backupCount": 5,
            "encoding":"utf8"
        },

        "app_file": {
            "level": Config.LOG_LEVEL,
            "class": "logging.handlers.TimedRotatingFileHandler",
            "formatter": "default",
            "filename": os.path.join(logdir,"app.log"),
            "when": "m",
            "interval": 1,
            "backupCount": 5,
            "encoding":"utf8"
        },
        
    },
    "loggers": {
        "defaultapp.diarios": {
            "handlers": ["console"] if debug else ["console", "diarios_file"],
            "level": Config.LOG_LEVEL,
            "propagate": False
        },
        "defaultapp.patrimonio": {
            "handlers": ["console"] if debug else ["console", "patrimonio_file"],
            "level": Config.LOG_LEVEL,
            "propagate": False
        },
        "defaultapp": {
            "handlers": ["console"] if debug else ["console", "app_file"],
            "level": Config.LOG_LEVEL,
            "propagate": False
        }
    },
    "root": {
        "level": "DEBUG" if debug else "INFO",
        "handlers": ["console"] 
    }
})

and I named my loggers following my flask python module names and inside these I take care of importing logging and calling getLogger(name):

import logging
logger = logging.getLogger(__name__)

so any log produced inside a particular module (.py file) will be written into the corresponding LOG file defined by in my handlers.

rgds

I am not sure why mine isn't working
I keep getting this error

and here is how my dictConfig looks like

@collinsmarra - Is it possibly because in the error_file handler, you specify a formatter called 'precise', but there is no such formatter in your formatters section?

@collinsmarra inside formatters you need to define precise or use default formatter rather than precise.