Skip to content

Instantly share code, notes, and snippets.

@pcdinh pcdinh/log.py
Created Feb 10, 2011

Embed
What would you like to do?
Python interpreter got crashed
import os
import time
from logging import basicConfig, getLogger, Formatter, StreamHandler
from logging.handlers import RotatingFileHandler
import multiprocessing, threading, logging, sys, traceback
class PipeLogHandler(logging.Handler):
"""This handler helps transfer log messages from child processes to the parent process via a pipe"""
shutdown = False
queue = None
def __init__(self, queue, handler):
logging.Handler.__init__(self)
self.queue = queue
self._handler = handler
self.polltime = 0.2
def setFormatter(self, fmt):
logging.Handler.setFormatter(self, fmt)
self._handler.setFormatter(fmt)
def send(self, s):
# self.queue.put_nowait(s)
# send just puts it in the queue for the server to retrieve
self.queue.put(s)
def _format_record(self, record):
# ensure that exc_info and args
# have been stringified. Removes any chance of
# unpickleable things inside and possibly reduces
# message size sent over the pipe
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
dummy = self.format(record)
record.exc_info = None
return record
def emit(self, record):
try:
s = self._format_record(record)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
raise
except:
self.handleError(record)
def close(self):
time.sleep(self.polltime+1) # give some time for messages to enter the queue
self._handler.close()
logging.Handler.close(self)
def __del__(self):
self.close() # hopefully this aids in orderly shutdown when things are going poorly.
class LogQueueReader(threading.Thread):
queue = None
shutdown = None
def __init__(self, queue):
self.queue = queue
self.shutdown = False
threading.Thread.__init__(self)
self.daemon = True
def run(self):
"""read from the queue and write to the log handlers
The logging documentation says logging is thread safe, so there
shouldn't be contention between normal logging (from the main
process) and this thread.
Note that we're using the name of the original logger.
"""
while (self.shutdown == False) or (self.queue.empty() == False):
try:
record = self.queue.get()
logger = logging.getLogger(record.name)
logger.callHandlers(record)
except (KeyboardInterrupt, SystemExit):
print "Quit signal"
self.shutdown = True
except EOFError:
break
except BaseException, e:
traceback.print_exc(file=sys.stderr)
print "Waiting for message from child processes"
time.sleep(0.1)
print "No message from child processes any more"
def get_logger(name=None, logdir=None, filename=None, level=None, maxBytes=0, backupCount=5, queue=None):
"""Gets a logger"""
logdir = os.path.abspath(logdir)
if not os.path.exists(logdir):
os.mkdir(logdir)
log = getLogger(name)
log.setLevel(level)
print os.path.join(logdir, filename)
handler = RotatingFileHandler(os.path.join(logdir, filename),
maxBytes=maxBytes,
backupCount=backupCount)
if queue is None:
queue = multiprocessing.Queue(-1)
# log_formatter = Formatter("%(asctime)s: %(levelname)s - %(name)s - %(process)s - %(message)s")
log_formatter = Formatter("%(asctime)s - %(message)s")
pipe_handler = PipeLogHandler(queue, handler)
pipe_handler.setFormatter(log_formatter)
log.addHandler(pipe_handler)
log.info("Logger %s initialized" % name)
return log
def test_child_process():
logger = logging.getLogger("parent")
logger.info("Child process ID %s" % os.getpid())
logger.info("Child process ID %s" % os.getpid())
print "Child process ID %s" % os.getpid()
if __name__ == '__main__':
# Logger for parent process
logger = get_logger(name="parent",
logdir="/tmp",
filename="parent.log",
level=logging.DEBUG,
maxBytes=1024*1024)
# Log message reader thread on parent process
log_queue_reader = LogQueueReader(multiprocessing.Queue(-1))
log_queue_reader.start()
try:
pids = []
for i in range(1, 5):
pid = os.fork()
if pid:
# parent
print "Parent PID %s" % os.getpid()
pids.append(pid)
logger.info("Parent process gets started")
else:
# child
print os.getpid()
test_child_process()
time.sleep(0.5)
print "Child process PID %s will exits now" % os.getpid()
os._exit(0)
except BaseException, e:
print str(e)
traceback.print_exc(file=sys.stdout)
try:
while pids:
try:
spid, status = os.waitpid(0, os.WNOHANG)
if spid > 0:
print "The child process %s exits" % spid
pids.remove(spid)
continue
print "No child process exits"
except OSError, e:
break
except BaseException, e:
print str(e)
finally:
time.sleep(0.1)
finally:
print pids
logging.shutdown()
print "Parent process exits"
sys.exit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.