sge_job_submission.py

#!/usr/bin/python3 env
# -*- coding: utf-8 -*-

"""Qsub command generator
    Author: Arup Ghosh 
    Creation date: 17/07/2019
    last modified: 18/07/2019
    Email: email [at] gmail.com
"""
import glob
import os

files=glob.glob("*.fastq.gz")
nodes=open("nodefile","r").read().split()
#print(nodes)
def _qsubFormat(fq,node):
    tool="/software/FastQC/fastqc -t 20 --outdir"
    cwd = os.getcwd()
    command="qsub -b y -q all.q@compute-0-{0} -N fastqc.{1} -e {1}_error.log -o {1}_cmd.log -V {3} {2}/test/ {2}/{1}".format(node,fq,cwd,tool)
    return command

#Assign one node to one file
if len(files) >len(nodes):
    #print("Split Jobs nodes {} files {}".format(len(nodes),len(files)))
    #Split the files list based upon the following calculation
    chunks=[files[i:i + 10] for i in range(0, len(files), 10)]
    for block in chunks:
        i=0
        for fq in block:
            i += 1
            #fix for node no tarting at compute-0-2
            nodeno = i+1
            print(_qsubFormat(fq,nodeno))
else:
    #print("Don't split jobs nodes {} files {}".format(len(nodes),len(files)))
    i=0
    for fq in files:
        i += 1
        #fix for node no tarting at compute-0-2
        nodeno = i+1
        print(_qsubFormat(fq,nodeno))