Some SLURM aliases

.bashrc

alias default_account='sacctmgr --parsable2 show user -s andrewpe | tail -1 | cut -f 2 -d '\''|'\'''
alias fairshare='sshare -a --accounts $(default_account)'
alias jhistory='sacct -u ${USER} --starttime $(date -d "-1 month" +%Y-%m-%d) --format=User,JobID,Jobname,partition,state,exitcode,time,start,end,elapsed,MaxRss,MaxVMSize,nnodes,ncpus,nodelist'
alias jshow='scontrol show job'
alias jinfo='sacct --starttime $(date -d "-1 month" +%Y-%m-%d) --format=User,JobID,Jobname%64,partition,qos,ReqCPUS,ReqMem,state,exitcode,time,start,end,elapsed,MaxRss,MaxVMSize,nnodes,ncpus,nodelist -j'
alias node_types='sinfo -o '\''%c %m %n %R'\'' --sort '\''P'\'''
alias qhogs='sreport cluster UserUtilizationByAccount -t hours'
alias si='sinfo -o "%20P %5D %14F %8z %10c %10m %10d %11l %16f %N"'
alias sq='squeue -o "%8i %.60j %4t %10u %20q %20a %10g %20P %10Q %5D %2C %5m %11l %11L %R"'
alias squ='squeue -u $USER --format="%.18i %.9P %.60j %.8u %.8a %.2t %.10M %.6D %R"'
alias sqr='squeue --me --states=R --format="%.18i %.9P %.48j %.8u %.8a %.2t %.10M %.6D %R"'
alias qos_limits="sacctmgr -p list qos $@ format=Name,Priority,GraceTime,MaxWall,MaxTRESPerUser,MaxJobsPU | column -ts'|'"
alias show_limits="scontrol show config | grep Max"
alias show_qos="sacctmgr show qos"
alias my_qos="sacctmgr -p list associations $@ format=Account,User,Partition,Qos,DefaultQOS tree | column -ts'|' | grep $USER"
alias partition_qos='scontrol show partition | awk -v RS= '\''{for(i=1;i<=NF;i++) if($i~/^PartitionName=|^AllowQos=|^DenyQos=/) printf "%s ", $i; print ""}'\'''

function jobshell() { srun --jobid $1 --overlap --pty bash; }
function jobtop() { srun --jobid $1 --overlap --pty top; }
function jobhtop() { srun --jobid $1 --overlap --pty htop; }

# Cancel ALL jobs matching the grep pattern for the current user
skill() {
    if [ -z "$1" ]; then
        echo "Usage: skill <grep_pattern>"
        return 1
    fi
    squeue -u "$USER" -h -o "%i %j" | grep "$1" | awk '{print $1}' | xargs -r scancel
    echo "Cancellation request sent for ALL jobs matching '$1'."
}

# Cancel RUNNING jobs matching the grep pattern for the current user
skillr() {
    if [ -z "$1" ]; then
        echo "Usage: skillr <grep_pattern>"
        return 1
    fi
    squeue -u "$USER" -t R -h -o "%i %j" | grep "$1" | awk '{print $1}' | xargs -r scancel
    echo "Cancellation request sent for RUNNING jobs matching '$1'."
}

# Cancel PENDING jobs matching the grep pattern for the current user
skillpd() {
    if [ -z "$1" ]; then
        echo "Usage: skillpd <grep_pattern>"
        return 1
    fi
    squeue -u "$USER" -t PD -h -o "%i %j" | grep "$1" | awk '{print $1}' | xargs -r scancel
    echo "Cancellation request sent for PENDING jobs matching '$1'."
}

user_jobs() {
    echo -e "USER            PENDING    RUNNING    CPUs(R)    MEM(R, GB)      GPUs(R)"
    echo -e "--------------- ---------- ---------- ---------- --------------- ----------"
    squeue -a -h -o "%u|%t|%C|%m|%b" | awk -F'|' '
    {
        user=$1; state=$2; cpus=$3; mem=$4; gres=$5;
        users[user]=1;
        
        if(state=="PD") {
            pend[user]++;
        } else if(state=="R") {
            run[user]++;
            tot_cpus[user]+=cpus;
            
            # Convert memory to GB
            m = mem;
            if(m ~ /G/) { gsub(/[^0-9.]/, "", m); tot_mem[user] += m; }
            else if(m ~ /M/) { gsub(/[^0-9.]/, "", m); tot_mem[user] += m/1024; }
            else if(m ~ /T/) { gsub(/[^0-9.]/, "", m); tot_mem[user] += m*1024; }
            else { gsub(/[^0-9.]/, "", m); tot_mem[user] += m/1024; }
            
            # Count GPUs from gres string (e.g., gpu:2, gpu:v100:1)
            g = 0;
            if (gres ~ /gpu/) {
                split(gres, parts, ",");
                for(i in parts) {
                    if(parts[i] ~ /gpu/) {
                        split(parts[i], gparts, ":");
                        num = gparts[length(gparts)];
                        if(num ~ /^[0-9]+$/) { g += num; }
                        else { g += 1; }
                    }
                }
            }
            tot_gpus[user]+=g;
        }
    }
    END {
        for(u in users) {
            printf "%-15s %-10d %-10d %-10d %-15.1f %-10d\n", u, pend[u]+0, run[u]+0, tot_cpus[u]+0, tot_mem[u]+0, tot_gpus[u]+0
        }
    }' | sort -k6,6nr -k4,4nr -k5,5nr
}