Skip to content

Instantly share code, notes, and snippets.

@kbfreder
Last active February 26, 2021 15:01
Show Gist options
  • Save kbfreder/848fd8a3aad48718d3eacaa284386d73 to your computer and use it in GitHub Desktop.
Save kbfreder/848fd8a3aad48718d3eacaa284386d73 to your computer and use it in GitHub Desktop.
basic logging
import logging
# REFERENCES
# logging message attributes: https://docs.python.org/3/library/logging.html#logrecord-attributes
# datetime formatting: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
# ---------------------------------------------
# BASIC LOGGING
# ---------------------------------------------
'''
Notes:
FORMATTING
- the `logging.basicConfig` format argument keyword is `format`;
- the `logging.Formatter` keyword is `fmt`.
- Below is a work-around to this annoying "feature"
ORDER:
- we must run logging.basicConfig to set log level *before* defining logger or making a call to logging
DON'T BOTHER:
- logger.setLevel() doesn't work
'''
msg_fmt = '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s'
date_fmt = '%Y-%m-%d %H:%M'
basic_format = {'format': msg_fmt, 'datefmt': date_fmt}
fmtr_format = {'fmt': msg_fmt, 'datefmt': date_fmt}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Test script for logging')
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose (debug) output', default=None)
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG, **basic_format)
else:
logging.basicConfig(level=logging.INFO, **basic_format)
logger = logging.getLogger(__name__)
logging.debug('Debug message')
logging.info('Info message')
logging.warning('Warning message')
# to write log messages to a file:
f_handler = logging.FileHandler('log_file.log')
f_handler.setLevel(logging.INFO)
f_handler.setFormatter(logging.Formatter(**fmtr_format))
logger.addHandler(f_handler)
logger.info('Info message that should go to file')
parser = argparse.ArgumentParser(description='Test script for logging')
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose (debug) output', default=None)
args = parser.parse_args()
main(args.verbose)
# ---------------------------------------------
# LOGGING IN PYSPARK
# ---------------------------------------------
# probably want to set spark logger ('py4j') to 'INFO', otherwise
# will see a lot of 'Command sent' / 'Answer received' messages
logging.basicConfig(level=logging.DEBUG, **basic_format)
logging.getLogger('py4j').setLevel(logging.INFO)
logger = logging.getLogger('pyspark')
logger.debug('Debug message')
logger.info('Info message')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment