wrapping python redis calls with exception handling

redis_wrapper.py

'''
Call a redis function several times with this wrapper if it has been observed that 
there are intermittent ResponseError exceptions and/or otherwise illegal return values.

Collect the good output and return that if it exists in some of the calls.

Example Usage:

rdb_cnxn = redis.ConnectionPool(host='localhost', port=6379, db=1)
rdb = redis.StrictRedis(connection_pool=rdb_cnxn)

# Original function call:
# fruits = rdb.lrange('my_fruits', 0, -1)
fruits = catch_redis_command_exceptions(
		rdb.lrange, {'illegal_outputs':[0,1]},
		'my_fruits', 0, -1
)


############################################
# Example report output with illegal outputs for lrange(): 

  1 {'args': ('values:2671a2ce', 0, -1),
  2  'crashes': [{'args': ('values:2671a2ce', 0, -1),
  3               'func': <bound method StrictRedis.lrange of StrictRedis<ConnectionPool<Connection<host=localhost,port=6379,d    b=1>>>>,
  4               'illegal_output': 1L,
  5               'kw': {}}],
  6  'func': <bound method StrictRedis.lrange of StrictRedis<ConnectionPool<Connection<host=localhost,port=6379,db=1>>>>,
  7  'good_output': ['8955',
  8                  '8955',
  9                  '7474',
 10                  '6379',
 11                  '6379'],
 12  'kw': {},
 13  'last_exception': None,
 14  'summary': {'crash': 1,
 15              'illegal_outputs': 1,
 16              'pass_fail_sequence': ['F',
 17                                     'P',
 18                                     'P',
 19                                     'P',
 20                                     'P',
 21                                     'P',
 22                                     'P',
 23                                     'P',
 24                                     'P',
 25                                     'P'],
 26              'report_uid': '51e5fa0f',
 27              'success': 9}}


############################################
# An example report for ResponseError('Protocol error: unbalanced quotes in request',)
# 
  1 {'args': ('ontologyclass_name_index', '6dd3e366'),
  2  'crashes': [{'args': ('ontologyclass_name_index', '6dd3e366'),
  3               'exception': ResponseError('Protocol error: unbalanced quotes in request',),
  4               'func': <bound method StrictRedis.hget of StrictRedis<ConnectionPool<Connection<host=localhost,port=6379,db=2>>>>,
  5               'kw': {},
  6               'stack_trace': ['Traceback (most recent call last):\n',
  7                               '  File "/Volumes/untitled/mystuff/staging/venvstage/lib/python2.7/site-packages/mypkg/utils/measure_redis_exceptions.py", line 34, in catch_redis_command_exceptions\n        output = func(*args, **kw)\n',
  8                               '  File "/Volumes/untitled/mystuff/staging/venvstage/lib/python2.7/site-packages/redis/client.py", line 1768, in hget\n    return self.execute_command(\'HGET\', name, k    ey)\n',
  9                               '  File "/Volumes/untitled/mystuff/staging/venvstage/lib/python2.7/site-packages/redis/client.py", line 529, in execute_command\n    return self.parse_response(connecti    on, command_name, **options)\n',
 10                               '  File "/Volumes/untitled/mystuff/staging/venvstage/lib/python2.7/site-packages/redis/client.py", line 541, in parse_response\n    response = connection.read_response(    )\n',
 11                               '  File "/Volumes/untitled/mystuff/staging/venvstage/lib/python2.7/site-packages/redis/connection.py", line 550, in read_response\n    raise response\n',
 12                               'ResponseError: Protocol error: unbalanced quotes in request\n']}],
 13  'func': <bound method StrictRedis.hget of StrictRedis<ConnectionPool<Connection<host=localhost,port=6379,db=2>>>>,
 14  'good_output': None,
 15  'kw': {},
 16  'last_exception': ResponseError('Protocol error: unbalanced quotes in request',),
 17  'summary': {'crash': 1,
 18              'illegal_outputs': 0,
 19              'pass_fail_sequence': ['F',
 20                                     'P',
 21                                     'P',
 22                                     'P',
 23                                     'P',
 24                                     'P',
 25                                     'P',
 26                                     'P',
 27                                     'P',
 28                                     'P'],
 29              'report_uid': '19452021',
 30              'success': 9}}



'''


import redis
import sys
import traceback
import pprint
import uuid

CODE_HOME = ''
LOG_DIR = 'logs'

def catch_redis_command_exceptions(func, parameters, *args, **kw):
	'''Call a redis function several times since it has been observed that 
	there are intermittent ResponseError exceptions and otherwise illegal return values.

	Collect the good output and return that if it exists in some of the calls.

	Both func and parameters are required.
	'''

	results = {'success':0, 'crash':0, 'illegal_outputs':0,
		'pass_fail_sequence':[],
		'report_uid': str(uuid.uuid4())[:8]
	}

	crashes = []

	# print 'starting...catch_redis_command_exceptions'
	e = None

	illegal_outputs = parameters.get('illegal_outputs', [])

	for attempt in range(10):
		try:
			output = func(*args, **kw)

			if output in illegal_outputs:
				results['crash'] += 1
				results['illegal_outputs'] += 1

				results['pass_fail_sequence'].append('F')

				crashes.append({
								'func':func,
								'args':args,
								'kw':kw,
								'illegal_output':output
				})

			else:
				results['success'] += 1
				results['pass_fail_sequence'].append('P')

				good_output = output
			
		except redis.exceptions.ResponseError, e:

			results['crash'] += 1
			results['pass_fail_sequence'].append('F')

			crashes.append({
							'func':func,
							'args':args,
							'kw':kw,
							'exception':e,
							# 'stack_trace':traceback.format_exc(),
							'stack_trace':traceback.format_exception(sys.exc_type, 
																	sys.exc_value, 
																	sys.exc_traceback),
			})
			# pprint.pformat()


	crash_output_summary = '''\
					\nAfter run of {func}, with args  \
					\n\t{args}, {kw}, \n\tsome results, \n\t{results} , and\
					\n\tLast exception "{e}".'''.format(**vars())

	# Detailed crash report for all attempts ....
	full_report = {
		'func':func,
		'last_exception':e,
		'args':args,
		'kw':kw,
		'summary':results,
		'crashes':crashes,
		'good_output':good_output
	}

	# Also write summary for main logs...
	if results['crash'] > 0 and results['success'] > 0: 
		sys.stderr.write(crash_output_summary)

		write_crash_report('crash.{}'.format(results['report_uid']), 
							pprint.pformat(full_report))

		return good_output 
	elif results['success'] == 0: 
		sys.stderr.write(crash_output_summary)

		write_crash_report('crash.{}'.format(results['report_uid']), 
							pprint.pformat(full_report))

		raise redis.exceptions.ResponseError, str(e)
	else: 
		#sys.stdout.write(crash_output_summary)
		#sys.stdout.write('okay done, no exceptions\n')
		return good_output 


def get_log_dir(log_file):
	''' Get the absolute path for a log file which will be placed in the ioq log dir 
		for this deployment.
	'''
	return os.path.join(CODE_HOME, LOG_DIR, log_file)


def write_crash_report(identifier, crash_data):

	report_filename = get_log_dir('{identifier}.{time_stamp}.txt'.format(
									time_stamp=strftime("%m%d%y_%H%M"),
									**vars()))

	with open(report_filename, 'w') as report_file_descriptor:
		report_file_descriptor.write(crash_data)