Skip to content

Instantly share code, notes, and snippets.

@FanchenBao
Last active December 10, 2019 01:11
Show Gist options
  • Save FanchenBao/9a6a76e970b5497346f60923f109b393 to your computer and use it in GitHub Desktop.
Save FanchenBao/9a6a76e970b5497346f60923f109b393 to your computer and use it in GitHub Desktop.
Expanded demo code for Python3 logging with multiprocessing
import logging
from logging import handlers
from time import sleep
def listener_configurer():
root = logging.getLogger()
file_handler = handlers.RotatingFileHandler('mptest.log', 'a', 300, 10)
console_handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s')
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
root.addHandler(file_handler)
root.addHandler(console_handler)
root.setLevel(logging.DEBUG)
def listener_process(queue):
listener_configurer()
while True:
while not queue.empty():
record = queue.get()
logger = logging.getLogger(record.name)
logger.handle(record)
sleep(1)
import logging
from logging import handlers
import multiprocessing
from listener import listener_process
from worker import worker_process
logger = logging.getLogger(__name__) # use module name
def root_configurer(queue):
h = handlers.QueueHandler(queue)
root = logging.getLogger()
root.addHandler(h)
root.setLevel(logging.DEBUG)
def main():
queue = multiprocessing.Queue(-1)
listener = multiprocessing.Process(
target=listener_process, args=(queue,))
listener.start()
root_configurer(queue)
logger.info('Logging from main')
workers = []
for i in range(3):
worker = multiprocessing.Process(target=worker_process, args=(queue,))
workers.append(worker)
worker.start()
for w in workers:
w.join()
logger.info('main function ends')
if __name__ == '__main__':
main()
import logging
from time import sleep
from random import random, randint
logger = logging.getLogger(__name__) # use module name
def worker_process(queue):
for i in range(3):
sleep(random())
logger.info(f'Logging a random number {randint(0, 10)}')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment