Python script to run a program a given number of times and check its return code for successful execution.

test_program.py

#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
import os
import subprocess
import argparse

def decode_retval(retval):
	retval_meanings = {
		139: 'segmentation fault',
	}
	try:
		return retval_meanings[retval]
	except KeyError:
		return 'unknown'

def main(program, num_tests, interactive=False):
	successes = 0
	failures = 0
	error_causes = {}

	for i in range(0, num_tests):
		command = '%s 2> /dev/null' % os.path.abspath(program)
		try:
			subprocess.check_output(command, shell=True)
		except subprocess.CalledProcessError, e:
			retval = e.returncode
			failures += 1
			cause = decode_retval(retval)
			try:
				error_causes[cause] += 1
			except KeyError:
				error_causes[cause] = 1
		else:
			successes += 1
		
		if interactive:
			if i % 10 == 0:
				sys.stdout.write(str(i))
			else:
				sys.stdout.write('.')
			sys.stdout.flush()
	if interactive:
		sys.stdout.write('%(num_tests)d\n\n' % locals())
	
	success_rate = (float(successes) / num_tests) * 100
	failure_rate = (float(failures) / num_tests) * 100
	sys.stdout.write('Total runs: %(num_tests)d\n' % locals())
	sys.stdout.write('Successes: %(successes)d (%(success_rate).2f%%)\n' % locals())
	sys.stdout.write('Failures: %(failures)d (%(failure_rate).2f%%)\n' % locals())
	sys.stdout.write('Error causes:\n')
	for error_cause, frequency in error_causes.items():
		percentage = (float(frequency) / failures) * 100
		sys.stdout.write('    %(error_cause)s: %(frequency)d (%(percentage).2f%%)\n' % locals())

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='Run a program a given number of times and check its return code for successful execution.')
	parser.add_argument('program', help='path to the program to be tested')
	parser.add_argument('num_tests', type=int, help='number of times to test the program')
	parser.add_argument('-i', '--interactive', action='store_true', help='show process indicator')
	
	args = parser.parse_args()
	main(args.program, args.num_tests, args.interactive)