spawn_compress.sh

#!/bin/bash

function ceil {
    echo "$1" | awk 'function ceil(v) { return (v == int(v)) ? v : int(v)+1 } { printf "%d", ceil($1) } '
}

input=$1
filename=$(basename $input)
linecount=$(wc -l $input | cut -f1 -d' ')

# maximum task count is 75000
# compute ceil(linecount / 75000); this represents the number of files that will be compressed in one run
batchSize=$(( linecount / 75000 ))
if [ $(( linecount % 75000 )) -ne 0 ]; then
    batchSize=$(( $batchSize + 1 ))
fi

# compute ceil(linecount / batchSize); this represents the maximum number of tasks in which we can split this directory
tasks=$(( linecount / batchSize ))
if [ $(( linecount % batchSize )) -ne 0 ]; then
    tasks=$(( $tasks + 1 ))
fi

qsub -q long -o "/broad/compbio/florinc/compress_out/${filename}.qsub.out" -cwd -t 1-$tasks -j y -b y -V "/broad/compbio/florinc/compress.sh $input $batchSize $tasks /broad/compbio/florinc/compress_out/${filename}"