Script to safely extend PBS job walltime. Use with check_reservation.py

extend_job

#!/bin/bash

# extend_job
#
# extend the life of a PBS job
#
# Written by Brett Pemberton, brett@vpac.org
# Copyright (C) 2008 Victorian Partnership for Advanced Computing

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

function check_res() {
   used_walltime=$(qstat -f $job | grep resources_used.walltime | awk '{print $3}')

   # If null, job hasn't started yet. Set to 0:0:0
   if [ -z $used_walltime ]; then
        used_walltime="0:0:0"
   fi

   # Reservation start time and number of nodes
   res_values=($(echo $showres | awk {'print $4 " " $7'} | cut -f 1 -d/))
   /root/bin/check_reservation.py ${res_values[@]} ${used_walltime} ${hours} ${walltime}

   if [ $? -ne 0 ]; then
      echo "Failed: check_reservation.py"
      exit 1
   fi
}

test=0

while getopts ":t" option
do
        case $option in
                t) test=1;;
        esac
done

shift $(($OPTIND - 1))

if [ $# -ne 2 ]
then
        echo "usage: $0 [-t] <job id> <hours to extend by>"
        exit 1
fi

job=$1
hours=$2

hours_check=$(echo $hours | egrep -vc "^[0-9]*$")

if [ $hours_check -ne 0 ]
then
        echo "ERROR: $hours is not a non-zero number"
        exit 1
fi

check=$(qstat -f $job)

if [ $? -ne 0 ]
then
        echo "ERROR: job id: $job is not valid"
        exit 1
fi

walltime=$(qstat -f $job | grep Resource_List.walltime | awk '{print $3}')
new_walltime=$(echo $walltime | awk -v h=$hours -F: '{print $1+h ":" $2 ":" $3}')
showres=$(showres | grep system)

# Check if system reservation was found.
if [ $? -eq 0 ]; then
   check_res
fi                

echo "will extend job $job by $hours hours"
echo "old walltime: $walltime"
echo "new walltime: $new_walltime"

if [ $test -eq 0 ]
then
        qalter -l walltime=$new_walltime $job
fi