Skip to content

Instantly share code, notes, and snippets.

@alexpearce
Created November 21, 2014 10:36
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save alexpearce/b416760472a605813efa to your computer and use it in GitHub Desktop.
Save alexpearce/b416760472a605813efa to your computer and use it in GitHub Desktop.
Method loaded by Ganga for automatically downloading job output
from os import makedirs
def download_lfns(job, path="/afs/cern.ch/work/a/apearce/Ganga"):
"""Downloads all Dirac files for all subjobs of a given job.
Keyword arguments:
job -- Ganga Job object. Only completed jobs/subjobs will be downloaded
path -- Prefix path to download to, will be suffixed by job_id/subjob_id
"""
print "Downloading LFNs to {0}/{1}".format(path, job.id)
# Collect all output files
if len(job.subjobs) > 0:
outputs = [
(sj.id, sj.outputfiles)
for sj in job.subjobs
if sj.status is "completed"
]
print "Will download {0}/{1} subjobs".format(
len(outputs), len(job.subjobs)
)
else:
outputs = [(0, job.outputfiles)] if job.status is "completed" else []
# Download all the DiracFile objects in each output file list
for jid, output in outputs:
for df in output.get(DiracFile):
if not df.lfn:
print "Skipping job {0} subjob {1}".format(
job.id,
jid
)
print "No output data LFN"
continue
local = "{0}/{1}/{2}".format(path, job.id, jid)
# Don't download to file if the dir exists
# Helpful if a job download is interrupted; just delete
# the failed subjobs and call this method again
try:
makedirs(local)
except OSError:
print "Skipping job {0} subjob {1}".format(
job.id,
jid
)
print "Directory exists: {0}".format(local)
continue
df.localDir = local
df.get()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment