Python interpreter got crashed
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import time | |
from logging import basicConfig, getLogger, Formatter, StreamHandler | |
from logging.handlers import RotatingFileHandler | |
import multiprocessing, threading, logging, sys, traceback | |
class PipeLogHandler(logging.Handler): | |
"""This handler helps transfer log messages from child processes to the parent process via a pipe""" | |
shutdown = False | |
queue = None | |
def __init__(self, queue, handler): | |
logging.Handler.__init__(self) | |
self.queue = queue | |
self._handler = handler | |
self.polltime = 0.2 | |
def setFormatter(self, fmt): | |
logging.Handler.setFormatter(self, fmt) | |
self._handler.setFormatter(fmt) | |
def send(self, s): | |
# self.queue.put_nowait(s) | |
# send just puts it in the queue for the server to retrieve | |
self.queue.put(s) | |
def _format_record(self, record): | |
# ensure that exc_info and args | |
# have been stringified. Removes any chance of | |
# unpickleable things inside and possibly reduces | |
# message size sent over the pipe | |
if record.args: | |
record.msg = record.msg % record.args | |
record.args = None | |
if record.exc_info: | |
dummy = self.format(record) | |
record.exc_info = None | |
return record | |
def emit(self, record): | |
try: | |
s = self._format_record(record) | |
self.send(s) | |
except (KeyboardInterrupt, SystemExit): | |
raise | |
except EOFError: | |
raise | |
except: | |
self.handleError(record) | |
def close(self): | |
time.sleep(self.polltime+1) # give some time for messages to enter the queue | |
self._handler.close() | |
logging.Handler.close(self) | |
def __del__(self): | |
self.close() # hopefully this aids in orderly shutdown when things are going poorly. | |
class LogQueueReader(threading.Thread): | |
queue = None | |
shutdown = None | |
def __init__(self, queue): | |
self.queue = queue | |
self.shutdown = False | |
threading.Thread.__init__(self) | |
self.daemon = True | |
def run(self): | |
"""read from the queue and write to the log handlers | |
The logging documentation says logging is thread safe, so there | |
shouldn't be contention between normal logging (from the main | |
process) and this thread. | |
Note that we're using the name of the original logger. | |
""" | |
while (self.shutdown == False) or (self.queue.empty() == False): | |
try: | |
record = self.queue.get() | |
logger = logging.getLogger(record.name) | |
logger.callHandlers(record) | |
except (KeyboardInterrupt, SystemExit): | |
print "Quit signal" | |
self.shutdown = True | |
except EOFError: | |
break | |
except BaseException, e: | |
traceback.print_exc(file=sys.stderr) | |
print "Waiting for message from child processes" | |
time.sleep(0.1) | |
print "No message from child processes any more" | |
def get_logger(name=None, logdir=None, filename=None, level=None, maxBytes=0, backupCount=5, queue=None): | |
"""Gets a logger""" | |
logdir = os.path.abspath(logdir) | |
if not os.path.exists(logdir): | |
os.mkdir(logdir) | |
log = getLogger(name) | |
log.setLevel(level) | |
print os.path.join(logdir, filename) | |
handler = RotatingFileHandler(os.path.join(logdir, filename), | |
maxBytes=maxBytes, | |
backupCount=backupCount) | |
if queue is None: | |
queue = multiprocessing.Queue(-1) | |
# log_formatter = Formatter("%(asctime)s: %(levelname)s - %(name)s - %(process)s - %(message)s") | |
log_formatter = Formatter("%(asctime)s - %(message)s") | |
pipe_handler = PipeLogHandler(queue, handler) | |
pipe_handler.setFormatter(log_formatter) | |
log.addHandler(pipe_handler) | |
log.info("Logger %s initialized" % name) | |
return log | |
def test_child_process(): | |
logger = logging.getLogger("parent") | |
logger.info("Child process ID %s" % os.getpid()) | |
logger.info("Child process ID %s" % os.getpid()) | |
print "Child process ID %s" % os.getpid() | |
if __name__ == '__main__': | |
# Logger for parent process | |
logger = get_logger(name="parent", | |
logdir="/tmp", | |
filename="parent.log", | |
level=logging.DEBUG, | |
maxBytes=1024*1024) | |
# Log message reader thread on parent process | |
log_queue_reader = LogQueueReader(multiprocessing.Queue(-1)) | |
log_queue_reader.start() | |
try: | |
pids = [] | |
for i in range(1, 5): | |
pid = os.fork() | |
if pid: | |
# parent | |
print "Parent PID %s" % os.getpid() | |
pids.append(pid) | |
logger.info("Parent process gets started") | |
else: | |
# child | |
print os.getpid() | |
test_child_process() | |
time.sleep(0.5) | |
print "Child process PID %s will exits now" % os.getpid() | |
os._exit(0) | |
except BaseException, e: | |
print str(e) | |
traceback.print_exc(file=sys.stdout) | |
try: | |
while pids: | |
try: | |
spid, status = os.waitpid(0, os.WNOHANG) | |
if spid > 0: | |
print "The child process %s exits" % spid | |
pids.remove(spid) | |
continue | |
print "No child process exits" | |
except OSError, e: | |
break | |
except BaseException, e: | |
print str(e) | |
finally: | |
time.sleep(0.1) | |
finally: | |
print pids | |
logging.shutdown() | |
print "Parent process exits" | |
sys.exit() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment