Last active
February 26, 2021 15:01
-
-
Save kbfreder/848fd8a3aad48718d3eacaa284386d73 to your computer and use it in GitHub Desktop.
basic logging
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import logging | |
# REFERENCES | |
# logging message attributes: https://docs.python.org/3/library/logging.html#logrecord-attributes | |
# datetime formatting: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior | |
# --------------------------------------------- | |
# BASIC LOGGING | |
# --------------------------------------------- | |
''' | |
Notes: | |
FORMATTING | |
- the `logging.basicConfig` format argument keyword is `format`; | |
- the `logging.Formatter` keyword is `fmt`. | |
- Below is a work-around to this annoying "feature" | |
ORDER: | |
- we must run logging.basicConfig to set log level *before* defining logger or making a call to logging | |
DON'T BOTHER: | |
- logger.setLevel() doesn't work | |
''' | |
msg_fmt = '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s' | |
date_fmt = '%Y-%m-%d %H:%M' | |
basic_format = {'format': msg_fmt, 'datefmt': date_fmt} | |
fmtr_format = {'fmt': msg_fmt, 'datefmt': date_fmt} | |
if __name__ == '__main__': | |
parser = argparse.ArgumentParser(description='Test script for logging') | |
parser.add_argument('-v', '--verbose', action='store_true', | |
help='Verbose (debug) output', default=None) | |
args = parser.parse_args() | |
if args.verbose: | |
logging.basicConfig(level=logging.DEBUG, **basic_format) | |
else: | |
logging.basicConfig(level=logging.INFO, **basic_format) | |
logger = logging.getLogger(__name__) | |
logging.debug('Debug message') | |
logging.info('Info message') | |
logging.warning('Warning message') | |
# to write log messages to a file: | |
f_handler = logging.FileHandler('log_file.log') | |
f_handler.setLevel(logging.INFO) | |
f_handler.setFormatter(logging.Formatter(**fmtr_format)) | |
logger.addHandler(f_handler) | |
logger.info('Info message that should go to file') | |
parser = argparse.ArgumentParser(description='Test script for logging') | |
parser.add_argument('-v', '--verbose', action='store_true', | |
help='Verbose (debug) output', default=None) | |
args = parser.parse_args() | |
main(args.verbose) | |
# --------------------------------------------- | |
# LOGGING IN PYSPARK | |
# --------------------------------------------- | |
# probably want to set spark logger ('py4j') to 'INFO', otherwise | |
# will see a lot of 'Command sent' / 'Answer received' messages | |
logging.basicConfig(level=logging.DEBUG, **basic_format) | |
logging.getLogger('py4j').setLevel(logging.INFO) | |
logger = logging.getLogger('pyspark') | |
logger.debug('Debug message') | |
logger.info('Info message') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment