Azure ML Management: use the Azure cli to check for running AML compute instances

amlcheck-currentsubscription.sh

#!/bin/bash

########################################################################################
#  Check for or stop all running Azure ML compute instances within the current         #
#  subscription or specified workspace                                                 #
#  * uses the azure resource graph to check for resources across resource groups       #
#  * assumes Azure CLI is installed and a default subscription is set (az account set) #
#  * use the -h option to see usage                                                    #
########################################################################################

subID="'$(az account show -o tsv --query 'id')'"
echo "Working in subscription $subID."

if [[ $1 == 'stop' ]]; then
   ACTION='stop'
   shift
elif [[ $1 == 'list' ]]; then 
   ACTION='list'
   shift
fi

TOTAL=0

while getopts ':hw:g:' arg; do
    case ${arg} in
      h ) 
        echo -e "\nUsage:"
        echo -e "    amlcheck -h                     Display this help message"
        echo -e "    amlcheck list                   List workspaces"
        echo -e "    amlcheck [-w ws -g rg]          List running compute instances [in specific workspace]"      
        echo -e "    amlcheck stop [-w ws -g rg]     Stop all compute instances [in workspace]\n"
        exit 0;;
      w ) ws=$OPTARG;;
      g ) rg=$OPTARG;;
      \?) echo "unknown option: -$arg" && exit 0;;
      : ) echo "-$arg requires an argument" && exit 0;;
    esac
done

# check if a workspace is specified
if [ -z $ws ]; then
    scope="subscription"

    echo -e "Checking for workspaces...\n"

    # use the azure resource graph to find all workspaces across resource groups
    wslist=$(az graph query -q "Resources | where subscriptionId =~ $subID and type =~ 'microsoft.machinelearningservices/workspaces' | order by resourceGroup asc | project name, resourceGroup" -o table)

    echo "> Found workspaces:"
    echo -e "$wslist\n"

else 
    scope="workspace"
    wslist="$ws $rg"
fi

# convert list to array and count items
wsarray=($(echo "$wslist" | awk '{if (NR>2) {print}}'))
c="${#wsarray[@]}"

if [[ $ACTION == 'list' ]]; then   
    # already done
    exit 0

# implement stopping all compute instances
elif [[ $ACTION == 'stop' ]]; then
    echo -e "Stop command received.\n"
    
    i=0
    while [ $i -lt $c ]; do
        ws="${wsarray[$i]}"
        rg="${wsarray[$((i+1))]}"
        echo -e "\nChecking $ws in resource group $rg for compute instances..."
        vmlist=$(az ml computetarget list -v -g $rg -w $ws -o tsv \
                --query "[?properties.computeType=='ComputeInstance' \
                         && properties.status.state=='Running']\
                         .[name]")
        if [ -z $vmlist ]; then
            echo -e "> No running compute in $ws.\n"
        else
            echo "> ${#vmlist[@]} instances to stop:"
            for i in $vmlist; do
                echo ">> stopping $i..."
                az ml computetarget computeinstance stop -v -g $rg -w $ws -n $i
            done
        fi
        ((i=i+2))
    done

# else just check for running instances
else
    echo -e "Checking $(($c/2)) workspaces for running compute instances...\n"
    i=0
    while [ $i -lt $c ]; do
        ws="${wsarray[$i]}"
        rg="${wsarray[$((i+1))]}"

        RESULT=$(az ml computetarget list -v \
        -w $ws \
        -g $rg \
        -o table \
        --query "[?properties.computeType=='ComputeInstance' && properties.status.state=='Running']\
                    .{Instance:name, \
                    State:properties.status.state, \
                    Owner:properties.status.createdBy.userName, \
                    Modified:properties.status.modifiedTime}")
        
        
        lines=$(echo "$RESULT" | wc -l)
        ci=$(($lines - 2))
        count=$(($ci>0?$ci:0))
        echo "> $count running compute instances in $ws ($rg) "
        echo -e "$RESULT\n"

        TOTAL=$(($TOTAL + $count))

        ((i=i+2))
    done

    echo -e "\n>> $TOTAL compute instances running in this $scope.\n"

    if [[ $TOTAL -gt 0 ]]; then
        echo -e "Run the following command to use this utility to stop all instances [in a specific workspace]:\n\t'amlcheck stop [-w <workspace> -g <resourcegroup>]'\n"
        echo -e "Run this Azure CLI command to stop a specific instance:\n\t'az ml computetarget computeinstance stop -v -w <workspace> -g <resourcegroup> -n <instance>' \n"
    fi
fi

amlcheck-default-rg.sh

#!/bin/bash

########################################################################################
#  Check for or stop all running Azure ML compute instances within the current         #
#  default subscription and resource group, or within a specified workspace in         #
#  that default group                                                                  #
#  * assumes Azure CLI is installed                                                    #
#  * assumes a default subscription is set (az account show)                           #
#  * assumes a default resource group is configured (az configure --list-defaults)     #
#  * use the -h option to see usage                                                    #
########################################################################################
sub=$(az account show -o tsv --query 'name')
rg=$(az configure --list-defaults -o tsv --query "[?name=='group'].value")

echo "Using defaults:"
echo -e "- subscription: \t\t$sub"
echo -e "- resource group: \t\t$rg\n"

if [[ $1 == 'stop' ]]; then
   ACTION='stop'
   shift
fi

TOTAL=0

while getopts ':hw:' arg; do
    case ${arg} in
      h ) 
        echo -e "Usage:"
        echo -e "    amlcheck -h                Display this help message"
        echo -e "    amlcheck [-w ws]           List running compute instances [in workspace]"      
        echo -e "    amlcheck stop [-w ws]      Stop all compute instances [in workspace]\n"
        exit 0;;
      w ) ws=$OPTARG;;
      \?) echo "unknown option: -$arg" && exit 0;;
      : ) echo "-$arg requires an argument" && exit 0;;
    esac
done

# check if a workspace is specified
if [ -z $ws ]; then
    scope="$sub, $rg"

    echo -e "Checking for workspaces...\n"

    wslist=$(az ml workspace list -o tsv --query [].workspaceName)

    echo "> Found workspaces:"
    echo -e "$wslist\n"
else 
    scope="$ws"
    wslist=$ws
    echo "Workspace $wslist"
fi

# implement stopping all compute instances
if [[ $ACTION == 'stop' ]]; then
    echo -e "Stop command received.\n"

    for ws in $wslist; do
        echo -e "\nChecking $ws for compute instances..."
        vmlist=$(az ml computetarget list -v -w $ws -o tsv \
                --query "[?properties.computeType=='ComputeInstance' \
                         && properties.status.state=='Running']\
                         .[name]")
        if [ -z $vmlist ]; then
            echo "> No running compute in $ws"
        else
            echo "> ${#vmlist[@]} instances to stop:"
            for i in $vmlist; do
                echo ">> stopping $i..."
                az ml computetarget computeinstance stop -v -w $ws -n $i
            done
        fi
    done

# else just list running instances
else     
    echo -e "Checking for running compute instances...\n"

    for ws in $wslist
    do
        RESULT=$(az ml computetarget list -v \
        -w $ws \
        -o table \
        --query "[?properties.computeType=='ComputeInstance' && properties.status.state=='Running']\
                    .{Instance:name, \
                    State:properties.status.state, \
                    Owner:properties.status.createdBy.userName, \
                    Modified:properties.status.modifiedTime}")
        
        
        lines=$(echo "$RESULT" | wc -l)
        ci=$(($lines - 2))
        count=$(($ci>0?$ci:0))
        echo "> $ws: $count running compute instances"
        TOTAL=$(($TOTAL + $count))
        echo -e "$RESULT\n"
    done
    echo -e "\n>> $TOTAL compute instances running in $scope.\n"


    if [[ $TOTAL -gt 0 ]]; then
        echo -e "Run the following command to use this utility to stop all instances [in a specific workspace]:\n\t'amlcheck stop [-w workspace]'\n"
        echo -e "Run this Azure CLI command to stop a specific instance:\n\t'az ml computetarget computeinstance stop -v -w <workspace> -n <instance>' \n"
    fi
fi

resource-graph-test.sh

# use the azure resource graph to find all workspaces across resource groups
wsarray=($(az graph query -q "Resources | where subscriptionId =~ 'cf48479b-e351-4599-8588-4867340afcce' and type =~ 'microsoft.machinelearningservices/workspaces' | project name, resourceGroup" -o tsv))

#wsarray=($wslist)

c="${#wsarray[@]}"
echo "$(($c/2)) workspaces found"
i=0
while [ $i -lt $c ]; do
    echo $i
    ws="${wsarray[$i]}"
    rg="${wsarray[$((i+1))]}"
    ((i=i+2))
done

test.sh

subID="'$(az account show -o tsv --query 'id')'"
#echo $sub


#subID="'cf48479b-e351-4599-8588-4867340afcce'"
echo "$subID"
wslist2=$(az graph query -q "Resources | where subscriptionId =~ $subID and type =~ 'microsoft.machinelearningservices/workspaces' | order by resourceGroup asc | project name, resourceGroup" -o tsv)
echo "$wslist2"

test2.sh

subID="'$(az account show -o tsv --query 'id')'"
echo "Working in subscription $subID."
  
wslist=$(az graph query -q "Resources | where subscriptionId =~ $subID and type =~ 'microsoft.machinelearningservices/workspaces' | order by resourceGroup asc | project name, resourceGroup" -o table)

echo "> Found workspaces:"
echo -e "$wslist\n"
echo "---"
echo -e $wslist
echo -e "---\n\n"

# ways of using the table output for display and then removing the first two lines later
new=$(echo "$wslist" | awk '$1 != "Name"' | awk '!/---/')
new2=$(echo "$wslist" | awk '{if (NR>2) {print}}')
echo -e "---\n\n"
echo "$new2"