basic logging

logging.py

import logging

# REFERENCES
# logging message attributes: https://docs.python.org/3/library/logging.html#logrecord-attributes
# datetime formatting: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior

# ---------------------------------------------
# BASIC LOGGING
# ---------------------------------------------

'''
Notes:
FORMATTING
- the `logging.basicConfig` format argument keyword is `format`; 
- the `logging.Formatter` keyword is `fmt`. 
- Below is a work-around to this annoying "feature"

ORDER:
- we must run logging.basicConfig to set log level *before* defining logger or making a call to logging

DON'T BOTHER:
- logger.setLevel() doesn't work
'''


msg_fmt = '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s'
date_fmt = '%Y-%m-%d %H:%M'

basic_format = {'format': msg_fmt, 'datefmt': date_fmt}
fmtr_format = {'fmt': msg_fmt, 'datefmt': date_fmt}



if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='Test script for logging')
	parser.add_argument('-v', '--verbose', action='store_true', 
						help='Verbose (debug) output', default=None)
	args = parser.parse_args()
  
	if args.verbose:
		logging.basicConfig(level=logging.DEBUG, **basic_format)
	else:
		logging.basicConfig(level=logging.INFO, **basic_format)
	
	logger = logging.getLogger(__name__)
	logging.debug('Debug message')
	logging.info('Info message')
	logging.warning('Warning message')
	
	# to write log messages to a file:
	f_handler = logging.FileHandler('log_file.log')
	f_handler.setLevel(logging.INFO)
	f_handler.setFormatter(logging.Formatter(**fmtr_format))
	logger.addHandler(f_handler)
	logger.info('Info message that should go to file')
  


	parser = argparse.ArgumentParser(description='Test script for logging')
	parser.add_argument('-v', '--verbose', action='store_true', 
						help='Verbose (debug) output', default=None)
	args = parser.parse_args()
	main(args.verbose)

# ---------------------------------------------
# LOGGING IN PYSPARK
# ---------------------------------------------

# probably want to set spark logger ('py4j') to 'INFO', otherwise
# will see a lot of 'Command sent' / 'Answer received' messages

logging.basicConfig(level=logging.DEBUG, **basic_format)
logging.getLogger('py4j').setLevel(logging.INFO) 
logger = logging.getLogger('pyspark')

logger.debug('Debug message')
logger.info('Info message')