basic example for using python to generate and submit template jobs

gistfile1.py

import subprocess

list_of_files = ['some', 'list', 'of', 'files', ...]

template = '''#!/bin/bash
# you can access python variables using named format strings
INPUT_FILES="%(input_files)s"

./run_analysis_program.py $INPUT_FILES
'''

# form the qsub command, note there is no input script specified
# as we are going to load it from stdin
cmd = ['qsub',
  '-o', '/dev/null',  # no log files
  '-e', '/dev/null',
  '-N', 'some job name',
  '-q', 'atlas',
  ]

# span the qsub command, and wire stdin to subprocess's pipe
job = subprocess.Popen(cmd,
  stdout=subprocess.PIPE,
  stderr=subprocess.STDOUT,
  stdin=subprocess.PIPE)

# bind the python variables in the template program.
# note that I turn the file list into a space-separated string
script = template % dict(input_files=' '.join(list_of_files))

# now write the script to qsub
response = job.communicate(script)[0]

if job.wait() != 0:
  print "Oops! qsub returned an error!"
  print "output:",response