sendJobs.py

import sys
import os
import numpy as np
#from file_323524 import *



## helper for files on dCache/EOS (LPC)
def useInputDir(inputDir, onEOS = True):
    theInputFiles = []
    for d in range(len(inputDir)):
        my_dir = inputDir[d]
        if not os.path.isdir(my_dir):
            print "ERROR: This is not a valid directory: ", my_dir
            if d==len(inputDir)-1:
                print "ERROR: No input files were selected"
                exit()
            continue
        print "Proceed to next directory"
        ls = os.listdir(my_dir)
        theInputFiles.extend([my_dir[:] + x for x in ls if x.endswith('root')])

    return theInputFiles


#runnumber = 325308
#inputdir = ["/eos/cms/store/data/Run2018E/ParkingL1SingleMu/RAW/v1/000/325/308/00000/"]
#runnumber = 325310
#inputdir = ["/eos/cms/store/data/Run2018E/ParkingL1SingleMu/RAW/v1/000/325/310/00000/"]
runnumber = 323524
inputdir = ["/eos/cms/store/data/Run2018D/SingleMuon/RAW/v1/000/323/524/00000/"]
input_list = useInputDir(inputdir)
eosdir = "/eos/user/t/tahuang/CSCEmulation/"
outputdir = os.path.join(eosdir, "Send_%d_ON/"%runnumber)
os.system("mkdir -p " + outputdir)
pwd = os.getcwd()

Nfiles = len(input_list)
files_per_job  = 2
nJobs = int(Nfiles/files_per_job)
if Nfiles % files_per_job != 0:
    nJobs += 1

file_obj2 = open(outputdir + "SendALL.sh",'w+')
email = "taohuang@tamu.edu"
for i in range(nJobs):
    # Write input file list
    last_index = ((i*files_per_job)+files_per_job)
    if last_index > len(input_list):
        last_index = len(input_list)
    files_list = input_list[ (i*files_per_job) : last_index ]
    input_name_tmp = os.path.join(outputdir, "input_" + str(i) + ".list")
    outputfile = os.path.join(outputdir, 'out_' + str(i) + '.root')
    file_obj = open(input_name_tmp,'w')
    for j, single_file in enumerate(files_list):
        file_obj.write('file:' + single_file + '\n')
    file_obj.close() 
    # Write bash file
    bash_name_tmp = "job_" + str(i) + ".sh"
    file_obj1 = open(outputdir + bash_name_tmp,'w')
    file_obj1.write('#!/bin/bash\n')
    file_obj1.write('cd '+ pwd + '\n')
    file_obj1.write('eval `scramv1 runtime -sh`\n')
    file_obj1.write('cmsRun CSCTPEmulator_cfg.py ' + input_name_tmp + ' /tmp/lcts_run_test_' + str(i) + '.root ' + outputfile +  '\n')
    file_obj1.close()
    # Write file who send all jobs
    #file_obj2.write('bsub -q 1nh -u youremail bash ' + outputdir + "/" + bash_name_tmp + '\n' )
    file_obj2.write('bsub -q 1nh -u '+ email +' bash ' + outputdir + "/" + bash_name_tmp + '\n' )
#    if i <= 1: 
#		file_obj2.write('bsub -q 1nh -u '+ email +' bash ' + outputdir + "/" + bash_name_tmp + '\n' )
#    else:
#		#file_obj2.write('bsub -q 8nm bash ' + outputdir + "/" + bash_name_tmp + '\n' )
#		file_obj2.write('bsub -q 1nh -u youremail bash ' + outputdir + "/" + bash_name_tmp + '\n' )
file_obj2.close()
file_obj3 = open(outputdir + "clean.sh",'w')
file_obj3.write('rm -rf out_*root\n')
file_obj3.write('rm -rf LSFJO*\n')
file_obj3.close()