general_paralleliser.py

import os, itertools
from subprocess import run
from multiprocessing.dummy import Pool
from functools import partial

def create_command_list(cli_program= " ", invocation=" ", script_name = " ", *args):
	""" 
	Constructs a list of arguments to be passed to cli_program and/or script name with the invocation specified. to be passed to run_in_parallel below
	example use:

	create_command_list('RScript', 'simulation.R', ' ', distributions)
	"""	
	commands = []
	combinations = list(itertools.product(*args)) 
	for combination in combinations:
		arguments = " ".join(combination)
		print(arguments)
		command = cli_program + invocation + script_name + ' ' + arguments
		commands.append(command)
	return commands	

def run_in_parallel(working_directory,commands, nprocesses=2):	
	""" 
	runs commands constructed using create_command_list in parallel by spawning number of processes specified in nprocesses separately
	"""	
	os.chdir(working_directory)
	pool = Pool(nprocesses)
	for i, returncode in enumerate(pool.imap(partial(run,shell=True),commands)):
		print(i, 'command executed, Returned', returncode )
	
if __name__ == '__main__':	
	commands = create_command_list('stata',' ', ' ', files,yvars,xvars)
  	run_in_parallel('C:/temp/',commands)