bargs: xargs for LSF compute farms.

bargs

#!/bin/bash


#  Tue Feb 17 16:18:06 GMT 2009 
# submit the command on the command line to a job array with parameters from stdin (a la xargs)

# For debugging
#set -o verbose 

# Die on unset variables
set -o nounset
# Die on errors
set -o errexit
# Die if any part of a pipe fails
set -o pipefail

LSBQUEUE="normal"
NEEDMEM=1000
NUMPROCS=1
JOBNAME=bargs
REPSTRING=""
STEPSIZE=1
DELIM=`echo -e "\n"`


usage()  {
    echo -e "usage:\n$0 [-J jobname] [-q queue] [-M mem] [-p procs] [-i replace-str]  [-h] command [initial-arguments]

-J jobname  Name for the job array (default ${JOBNAME})
-q queue    Queue to put the jobs in (default ${LSBQUEUE})
-M mem      Requested maximum memory consumption in megabytes (default ${NEEDMEM})
-n procs    Number of processors requested (default ${NUMPROCS})
-w cond     Wait until condition (see bsub).
-i replace-str  String to replace with the input names
-L max-lines    Uses max-lines input lines for each job.
-h          Show this message and exit." >&2
    exit 1

}



while getopts  "J:q:hn:M:i:L:w:" flag
do
  case "$flag" in 
      i)
      REPSTRING="$OPTARG"
      ;;
      d)
      DELIM=`echo -e "$OPTARG"` # Currently doesn't work.
      ;;
      J)
      JOBNAME="$OPTARG"
      ;;
      q)
      LSBQUEUE="$OPTARG"
      ;;
      M)
      NEEDMEM="$OPTARG"
      ;;
      L)
      STEPSIZE="$OPTARG"
      ;;
      w)
      WAITCOND="-w$OPTARG"
      ;;
      n)
      NUMPROCS="$OPTARG"
      ;;
      h|*)
      usage
      ;;
  esac
done
shift $((OPTIND-1)); OPTIND=1


if (( ${#} < 1 )); then
	    usage
fi



declare -a PARAMS=( )

while  read  subPARAMS; do 
    PARAMS[${#PARAMS[@]}]=`printf "%q" "${subPARAMS}"`
done ; 



#cat <<-EOF
bsub ${WAITCOND:-}  -q "$LSBQUEUE" -J "${JOBNAME}[1-${#PARAMS[@]}:${STEPSIZE}]" -n${NUMPROCS} -o $PWD/${JOBNAME}.%J.%I.out -e $PWD/${JOBNAME}.%J.%I.err -R "rusage[mem=${NEEDMEM}] select[mem>${NEEDMEM}] span[ptile=${NUMPROCS}]" -M$(( ${NEEDMEM} * 1000 )) <<-EOF
#!/bin/bash
# Die on unset variables
  set -o nounset
# Die on errors
  set -o errexit
# Die if any part of a pipe fails
  set -o pipefail
# For debugging
#set -o verbose 
#set -o errtrace


  # This part may be fragile!!!
  ALLFILES=( ${PARAMS[@]} )
  BASEidx=\$(( \${LSB_JOBINDEX:-1} - 1 ))
  for STEPidx in \`seq ${STEPSIZE}\` 
  do
    FILEidx=\$(( \${BASEidx} + \${STEPidx} - 1 ))
  

    READFILE="\${ALLFILES[\${FILEidx}]}"
    if [ ${#REPSTRING} -gt 0 ]; 
	then
	RUNCMD=( '$@' )
	RUNCMD2=( "\${RUNCMD[@]//${REPSTRING}/\${READFILE}}" )
	#RUNCMD2=( \${RUNCMD[@]//${REPSTRING}/\${READFILE}} )
	#echo \${RUNCMD2[@]}
	#echo "RUNNING"
	#"\${RUNCMD2[@]}"
	bash  -c "\${RUNCMD2[@]}"
	#echo "DONE"
    else
	$@ "\${READFILE}"
    fi
  done
  exit 0


EOF

qargs

#!/bin/bash


#  Tue Feb 17 16:18:06 GMT 2009 
# submit the command on the command line to a job array with parameters from stdin (a la xargs)

# For debugging
#set -o verbose 

# Die on unset variables
set -o nounset
# Die on errors
set -o errexit
# Die if any part of a pipe fails
set -o pipefail

NEEDMEM=100
NUMPROCS=1
JOBNAME=$(basename $0)
REPSTRING=""
STEPSIZE=1
DELIM=`echo -e "\n"`
WORKDIR=$PWD

usage()  {
    echo -e "usage:\n$0 [-J jobname] [-q queue] [-M mem] [-p procs] [-i replace-str]  [-h] command [initial-arguments]

-J jobname  Name for the job array (default ${JOBNAME})
-q queue    Queue to put the jobs in (default ${QUEUE:-})
-M mem      Requested maximum memory consumption in megabytes (default ${NEEDMEM})
-n procs    Number of processors requested (default ${NUMPROCS})
-W cond     Wait until condition (see qsub).
-d workdir  Set current working directory for jobs (default $WORKDIR).
-i replace-str  String to replace with the input names
-L max-lines    Uses max-lines input lines for each job.
-h          Show this message and exit." >&2
    exit 1

}



while getopts  "J:q:hn:M:i:L:d:W:" flag
do
  case "$flag" in 
      i)
      REPSTRING="$OPTARG"
      ;;
      d)
      DELIM=`echo -e "$OPTARG"` # Currently doesn't work.
      ;;
      J)
      JOBNAME="$OPTARG"
      ;;
      q)
      QUEUE="-q $OPTARG"
      ;;
      M)
      NEEDMEM="$OPTARG"
      ;;
      L)
      STEPSIZE="$OPTARG"
      ;;
      d)
	  WORKDIR="$OPTARG"
	  ;;
      W)
      WAITCOND="-W $OPTARG"
      ;;
      n)
      NUMPROCS="$OPTARG"
      ;;
      h|*)
      usage
      ;;
  esac
done
shift $((OPTIND-1)); OPTIND=1


if (( ${#} < 1 )); then
	    usage
fi



declare -a PARAMS=( )

while  read  subPARAMS; do 
    PARAMS[${#PARAMS[@]}]=`printf "%q" "${subPARAMS}"`
done ; 



#cat <<-EOF
qsub ${WAITCOND:-} ${QUEUE:-} -N "${JOBNAME}" -t "1-${#PARAMS[@]}%${STEPSIZE}"  \
    -l "mem=${NEEDMEM}mb,nodes=1:ppn=${NUMPROCS}"  -d "$WORKDIR" <<-EOF
#!/bin/bash
# Die on unset variables
  set -o nounset
# Die on errors
  set -o errexit
# Die if any part of a pipe fails
  set -o pipefail
# For debugging
#set -o verbose 
#set -o errtrace


  # This part may be fragile!!!
  ALLFILES=( ${PARAMS[@]} )
  BASEidx=\$(( \${PBS_ARRAYID:-1} - 1 ))
  for STEPidx in \`seq ${STEPSIZE}\` 
  do
    FILEidx=\$(( \${BASEidx} + \${STEPidx} - 1 ))
  

    READFILE="\${ALLFILES[\${FILEidx}]}"
    if [ ${#REPSTRING} -gt 0 ]; 
	then
	RUNCMD=( '$@' )
	RUNCMD2=( "\${RUNCMD[@]//${REPSTRING}/\${READFILE}}" )
	#RUNCMD2=( \${RUNCMD[@]//${REPSTRING}/\${READFILE}} )
	#echo \${RUNCMD2[@]}
	#echo "RUNNING"
	#"\${RUNCMD2[@]}"
	bash  -c "\${RUNCMD2[@]}"
	#echo "DONE"
    else
	$@ "\${READFILE}"
    fi
  done
  exit 0


EOF