Skip to content

Instantly share code, notes, and snippets.

@natefoo
Created April 15, 2020 14:27
Show Gist options
  • Save natefoo/07b2ae0b9dc968bd9003841c95f0368c to your computer and use it in GitHub Desktop.
Save natefoo/07b2ae0b9dc968bd9003841c95f0368c to your computer and use it in GitHub Desktop.
if options.requeue_job:
from galaxy.util import directory_hash_id
job_id = options.requeue_job
job = model.context.current.query(model.Job).enable_eagerloads(False).get(job_id)
job_dir_hash = os.path.join(*directory_hash_id(job_id))
job_working_dir = os.path.join(JOB_WORKING_DIR, job_dir_hash, str(job_id))
print('Attempting to requeue job %s/%s' % (job_id, job.job_runner_external_id))
if os.path.exists(job_working_dir):
cleared_dir = os.path.join(JOB_WORKING_DIR, '_cleared_contents', job_dir_hash, str(job_id))
if not os.path.exists(cleared_dir):
os.makedirs(cleared_dir)
dest = os.path.join(cleared_dir, datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
print('Moving %s to %s' % (job_working_dir, dest))
shutil.move(job_working_dir, dest)
os.makedirs(job_working_dir)
else:
print('Did not find job working directory at %s' % job_working_dir)
sys.exit(1)
for hda in [ jtod.dataset for jtod in job.output_datasets ]:
jeom = model.context.current.query(model.JobExternalOutputMetadata).filter_by(job_id=job_id, history_dataset_association_id=hda.id, is_valid=True).first()
while jeom:
print('Invalidating JEOM %s' % jeom.id)
jeom.is_valid = False
model.context.current.add(jeom)
model.context.current.flush()
jeom = model.context.current.query(model.JobExternalOutputMetadata).filter_by(job_id=job_id, history_dataset_association_id=hda.id, is_valid=True).first()
slurm_id = job.job_runner_external_id
job.destination_id = None
job.destination_params = None
job.job_runner_name = None
job.job_runner_external_id = None
job.info = 'This job was requeued (in an attempt to allow it to run sooner), please excuse the interruption and contact Galaxy support with any problems'
job.set_state(model.Job.states.NEW)
model.context.current.add(job)
model.context.current.flush()
print('Job %s reset to new state' % job_id)
slurm_id, cluster = (slurm_id.split('.') + [None])[:2]
cmd = ['scancel']
if cluster:
cmd.extend(['-M', cluster])
cmd.append(slurm_id)
print('Cancelling old job with: %s' % cmd)
subprocess.check_call(cmd)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment