Logging into separate files for multiprocessing and tornado/flask in python

flask_app.py

# create flask app to be run by tornado process

# process-specific globals
import global_vars 

from flask import Flask, request, Response, json, abort, jsonify

# oridnary (non-flask) json if needed
import json as json2
 
# from deeper_module import do_something

app = Flask(__name__)
app.config.from_object(__name__)

# get process-specific logger
# do such import in any submodule !
# as global_vars is changed on process fork!
rootLogger = global_vars.multiprocess_globals["logger"]
logger = rootLogger.getChild(__name__)


def create_app():
    
    app = Flask(__name__)
    
    @app.route('/my_url1', methods=['POST'])
    def my_url1():
        
        body = json.loads(request.data)
        
        # debug to process-specific logger
        logger.debug(body)
        
        # do_something()
        
        response = app.response_class(
            response=json.dumps({'response':'good'}),
            status=200,
            mimetype='application/json'
        )
        return response
    
    # another urls
    
    return app

global_vars.py

# clobal variables to be used across processes.
# Separate file is needed to make globals accessible from different submodules.

# Global variables which need to exist in the scope of each process.
# Variables are added into this dictionary during process initialization.
multiprocess_globals = {}

mainprogram.py

# program runs multiple torando servers with flask apps concurrently
# logging is done into common stdout.log in 'logs' subfolder
# and into separate logs of different processes
# In each submodule, the following import of logger must be made:

'''
import global_vars
rootLogger = global_vars.multiprocess_globals["logger"]
logger = rootLogger.getChild(__name__)
'''
# launching threads and child processes is supported!
# in this case you will see in log something like
# [Process-3:1]

import logging
import os
import multiprocessing

from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.options import define, options

# a way to pass variables into separate modules, 
# process-specific because of 'fork' mode
# (each process will have its own version of this module)
import global_vars



logPath = os.environ.get('LOGFOLDER','logs')
fileName = os.environ.get('LOGFILE', "stdout.log")

address = os.environ.get('ADDRESS','0.0.0.0')
port = os.environ.get('PORT','8888')

NUM_PROCESSES = os.cpu_count()

# initializes the main logger to be used across all modules

logFormatter = logging.Formatter("%(asctime)s [%(processName)-12.12s] [%(threadName)-12.12s] [%(levelname)-5.5s] [%(filename)s:%(lineno)d] %(message)s")
rootLogger = logging.getLogger(__name__)

# first handler is general log
fileHandler = logging.FileHandler("{0}/{1}".format(logPath, fileName))
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
# second handler is logging to console
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler) 

rootLogger.setLevel("DEBUG") # log everything
rootLogger.propagate = False

# until process branches, it uses rootLogger
global_vars.multiprocess_globals["logger"] = rootLogger



# third handler is process-specific log, initialized in processes
def init_logger2(secondary_logfile, rootLogger):
    fileHandler1 = logging.FileHandler("{0}/{1}".format(logPath, 'process_'+str(secondary_logfile)+'.log'))
    fileHandler1.setFormatter(logFormatter)
    rootLogger.addHandler(fileHandler1)

    return rootLogger



# external modules import goes here!
# otherwise they will not find any logger!
from flask_app import create_app




# ---------------

# process function
def run(process_id):
    
    # initialize process-specific logger
    processLogger = init_logger2(process_id, rootLogger)
    global_vars.multiprocess_globals["logger"] = processLogger
    
    # here you can run tornado app:
    
    try:
        app = create_app()  # pass interests to flask app        
        ioloop = IOLoop()
        http_server_api = HTTPServer(WSGIContainer(app))
        
        # reuse_port allows multiple servers co-exist
        # as separate processes
        http_server_api.bind(address=address, port=port, reuse_port=True) 
        http_server_api.start()
        
        processLogger.info("Process %s started %s:%s" % (process_id, address,
                                            port))

        ioloop.start()
    except Exception as e:
        processLogger.error(e)
    
    
    


# start processes (tornado servers)
if __name__ == '__main__':

    processes = []
    for i in range(1,NUM_PROCESSES):
        p = multiprocessing.Process(target=run, args=(str(i),))
        p.daemon = False # if we want to spawn child processes
        #p.daemon = True # if we want to gracefully stop program
        processes.append(p)
    
    
    # Run processes:
    
    for p in processes:
        p.start()
        
    # block program from exiting
    for p in processes:
        p.join() 
    

test.sh

curl -H "Content-Type: application/json" -X POST -d '{"bla-bla":"bla"}' 127.0.0.1:8888/my_url1