requeue-job.py

if options.requeue_job:
    from galaxy.util import directory_hash_id
    job_id = options.requeue_job
    job = model.context.current.query(model.Job).enable_eagerloads(False).get(job_id)
    job_dir_hash = os.path.join(*directory_hash_id(job_id))
    job_working_dir = os.path.join(JOB_WORKING_DIR, job_dir_hash, str(job_id))
    print('Attempting to requeue job %s/%s' % (job_id, job.job_runner_external_id))
    if os.path.exists(job_working_dir):
        cleared_dir = os.path.join(JOB_WORKING_DIR, '_cleared_contents', job_dir_hash, str(job_id))
        if not os.path.exists(cleared_dir):
            os.makedirs(cleared_dir)
        dest = os.path.join(cleared_dir, datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
        print('Moving %s to %s' % (job_working_dir, dest))
        shutil.move(job_working_dir, dest)
        os.makedirs(job_working_dir)
    else:
        print('Did not find job working directory at %s' % job_working_dir)
        sys.exit(1)
    for hda in [ jtod.dataset for jtod in job.output_datasets ]:
        jeom = model.context.current.query(model.JobExternalOutputMetadata).filter_by(job_id=job_id, history_dataset_association_id=hda.id, is_valid=True).first()
        while jeom:
            print('Invalidating JEOM %s' % jeom.id)
            jeom.is_valid = False
            model.context.current.add(jeom)
            model.context.current.flush()
            jeom = model.context.current.query(model.JobExternalOutputMetadata).filter_by(job_id=job_id, history_dataset_association_id=hda.id, is_valid=True).first()
    slurm_id = job.job_runner_external_id
    job.destination_id = None
    job.destination_params = None
    job.job_runner_name = None
    job.job_runner_external_id = None
    job.info = 'This job was requeued (in an attempt to allow it to run sooner), please excuse the interruption and contact Galaxy support with any problems'
    job.set_state(model.Job.states.NEW)
    model.context.current.add(job)
    model.context.current.flush()
    print('Job %s reset to new state' % job_id)
    slurm_id, cluster = (slurm_id.split('.') + [None])[:2]
    cmd = ['scancel']
    if cluster:
        cmd.extend(['-M', cluster])
    cmd.append(slurm_id)
    print('Cancelling old job with: %s' % cmd)
    subprocess.check_call(cmd)