job submission throttling for MPComplete on XSEDE Comet based on SU usage

submit_jobs_comet.sh

#!/bin/bash -l

source /home/phuck/mp_prod/bin/activate
export FW_CONFIG_FILE=$FW_CONFIG_mp_prod
export DB_LOC=/home/phuck/mp_prod/config/dbs
export VENV_LOC=/home/phuck/mp_prod/bin/activate
export SCRIPT_LOC=/home/phuck/mp_prod/config/scripts


# xdusage doesn't work in a cronjob due to sudo usage, needs tty!?
today=`date +%Y-%m-%d`
alloc_period_string="2016-01-01/2016-12-31" #`xdusage -r comet -u phuck | grep "Allocation:" | awk {'print $2'}`
alloc_period=(${alloc_period_string//\// })
alloc_sus=250000 #`xdusage -r comet -u phuck | grep "Total=" | awk {'print $2'} | cut -d= -f2 | sed 's:,::g'`
days_remaining=$(( (`date -d ${alloc_period[1]} +%s` - `date -d "00:00" +%s`) / (24*3600) ))

function get_sus_used() {
    # $1 = start date as YYYY-MM-DD
    local sus_used=0
    for i in `sacct -u phuck -S $1 -n -o User,elapsed | grep phuck | awk {'print $2'}`; do
        local dash_split=(${i//-/ })
        if [ ${#dash_split[@]} -eq 1 ]; then
            local colon_split=(${dash_split[0]//:/ })
            local days=0
        else
            local colon_split=(${dash_split[1]//:/ })
            local days=${dash_split[0]}
        fi
        local sus=$(( 10#${days} * 24 * 60 + 10#${colon_split[0]} * 60 + 10#${colon_split[1]} + 1 ))
        let sus_used+=$sus
    done
    echo $sus_used
}

sus_used_total=`get_sus_used ${alloc_period[0]}`
sus_remaining=$(( $alloc_sus - $sus_used_total ))
sus_target=$(( $sus_remaining / $days_remaining * 6 / 5 )) # go 20% over
sus_used_today=`get_sus_used $today`
sus_for_today=$(( $sus_target - $sus_used_today ))

now=`date "+%Y-%m-%d %H:%M:%S"`
if [ $sus_for_today -gt 0 ]; then
    echo "$now INFO $sus_for_today SUs remaining for today. Submitting more jobs ..."
    cd /oasis/scratch/comet/phuck/temp_project
    python /home/phuck/mp_prod/codes/MPWorks/mpworks/fix_scripts/fix_mpcomplete.py
    qlaunch -r rapidfire --nlaunches 0 -m 25 --sleep 100 -b 10000
else
    echo "$now INFO $sus_used_today SUs used today. Over limit of $sus_target SUs. No more jobs submitted today."
fi