Stress test hotplug and idle with the power of randomness

cpuidle_stress.sh

#!/bin/bash

## Author: Pratik Rajesh Sampat
## Usage: Control either by duration (-d seconds ) or iterations (-i iterations)
## Default runtime: 10 mins

NUM_CPUS=$(nproc --all)
NUM_ITER=1
SMT=4
EXEC_TIME=600 # In Seconds = 10 mins
ITER=1
is_exec=false
is_iter=false

helpme()
{
	printf "Usage: $0 [-h] [-ti args]
	[-h <help>]
	[-t <Time of execution in seconds>]
	[-i <Number of iterations>]
    Either use time or number of iterations to control the experiment
	\n"
	exit 2
}

parse_arguments()
{
	while getopts "ht:t:i:" arg
	do
		case $arg in
			h) # --help
				helpme
				;;
			t) # Execution time
				EXEC_TIME=$OPTARG
                is_exec=true
				;;
			i) # Number of Iterations
				ITER=$OPTARG
                is_iter=true
				;;
			\? )
				helpme
				;;
		esac
	done
}

# random permutation of input
declare -a shuf_list=()
rand_perm() {
	# make the input an array
	local -a items=( "$@" )
	# all the indices of the array
	local -a items_arr=( "${!items[@]}" )
	# loop while there is at least one index
	while [ ${#items_arr[@]} -gt 0 ]; do
		# pick a random number between 1 and the length of the indices array
		local rand=$(( RANDOM % ${#items_arr[@]} ))
		# get the item index from the array of indices
		local items_idx=${items_arr[$rand]}
		# append that item to the out array
		shuf_list+=("${items[$items_idx]}")
		### NOTE array is not reindexed when pop'ing, so we redo an array of 
		### index at each iteration
		# pop the item
		unset "items[$items_idx]"
		# recreate the array
		items_arr=( "${!items[@]}" )
	done
}

hotplug_test() {
    echo "---Starting hotplug test---"
    rand_perm "${first_cpu_list[@]}"
    local cpu_list=("${shuf_list[@]}")
    for ((iter=0; iter<ITER; iter++))
    do
        echo "--Iteration $iter--"
        for ((cpu=0; cpu<NUM_CPUS/SMT; cpu++))
        do
            unset shuf_list
            ## Shuffle the threads
            for ((thread=${cpu_list[$cpu]}; thread<${cpu_list[$cpu]}+SMT; thread++))
            do
                idle_thread_list+=($thread)
            done
            rand_perm "${idle_thread_list[@]}"
            idle_thread_list=("${shuf_list[@]}")
            unset shuf_list

            ## Offline the threads
            for ((thread=0; thread<SMT; thread++))
            do
                echo 0 > /sys/devices/system/cpu/cpu${idle_thread_list[$thread]}/online
                if (( $? != 0)); then
                    echo "Offline for CPU ${idle_thread_list[$thread]} failed"
                    exit 2
                fi
            done

            sleep 2

            ## Online the threads
            for ((thread=0; thread<SMT; thread++))
            do
                echo 1 > /sys/devices/system/cpu/cpu${idle_thread_list[$thread]}/online
                if (( $? != 0)); then
                    echo "Online for CPU ${idle_thread_list[$thread]} failed"
                    exit 2
                fi
            done

            echo "HOTPLUG TEST OK: CORE=${cpu_list[$cpu]}"
            unset idle_thread_list

            elapsed_time=$(date +%s)
            if [ "$is_exec" = true ] && (( $elapsed_time - $start_time > $EXEC_TIME ));then
                    echo "Time has elapsed"
                    exit 0
            fi
        done
        unset cpu_list
        rand_perm "${first_cpu_list[@]}"
        cpu_list=("${shuf_list[@]}")
    done
}

cpuidle_test() {
    echo "---Starting cpuidle test---"

    declare -a idle_usage_list=()
    declare -a idle_thread_list=()

    for ((iter=0; iter<ITER; iter++))
    do
        echo "--Iteration $iter--"
        for ((state=1; state<idle_states; state++))
        do
            rand_perm "${first_cpu_list[@]}"
            local cpu_list=("${shuf_list[@]}")
            for ((cpu=0; cpu<NUM_CPUS/SMT; cpu++))
            do
                unset shuf_list
                # echo ${cpu_list[$cpu]}
                # echo "thread"

                ## Shuffle the threads
                for ((thread=${cpu_list[$cpu]}; thread<${cpu_list[$cpu]}+SMT; thread++))
                do
                    idle_thread_list+=($thread)
                done
                rand_perm "${idle_thread_list[@]}"
                idle_thread_list=("${shuf_list[@]}")
                unset shuf_list

                ## Record the initial usage values
                for ((thread=0; thread<SMT; thread++))
                do
                    usage=$(cat /sys/devices/system/cpu/cpu${idle_thread_list[$thread]}/cpuidle/state$state/usage)
                    idle_usage_list+=($usage)
                    # echo "Curr usage $usage, CPU: ${idle_thread_list[$thread]} state: $state"
                done

                ## Enable the state for this core and wait
                for ((thread=0; thread<SMT; thread++))
                do
                    # echo "Enabling Idle for thread ${idle_thread_list[$thread]}"
                    echo 0 > /sys/devices/system/cpu/cpu${idle_thread_list[$thread]}/cpuidle/state$state/disable
                    if (( $? != 0)); then
                        echo "Enable idle for CPU ${idle_thread_list[$thread]} failed"
                        exit 2
                    fi
                done
                sleep 5
                usage_idx=0

                ## Disable the state for this core and compare the usage
                for ((thread=0; thread<SMT; thread++))
                do
                    # echo "Disable Idle for thread ${idle_thread_list[$thread]}"
                    echo 1 > /sys/devices/system/cpu/cpu${idle_thread_list[$thread]}/cpuidle/state$state/disable
                    if (( $? != 0)); then
                        echo "Disable idle for CPU ${idle_thread_list[$thread]} failed"
                        exit 2
                    fi
                    
                    usage=$(cat /sys/devices/system/cpu/cpu${idle_thread_list[$thread]}/cpuidle/state$state/usage)
                    
                    if (( $usage <= ${idle_usage_list[$usage_idx]} )); then
                        # echo "Prev usage: ${idle_usage_list[$usage_idx]}, Curr usage $usage, CPU: ${idle_thread_list[$thread]} state: $state"
                        echo "Prev usage: ${idle_usage_list[$usage_idx]}, Curr usage $usage"
                        echo "Usage stats not increasing."
                        echo "CPUIDLE TEST FAIL: CORE=${cpu_list[$cpu]} STATE=$state"
                        exit 2
                    fi
                    usage_idx=$((usage_idx+1))
                done
                unset idle_usage_list
                echo "CPUIDLE TEST OK: CORE=${cpu_list[$cpu]} STATE=$state"
                unset idle_thread_list
                elapsed_time=$(date +%s)

                if [ "$is_exec" = true ] && (( $elapsed_time - $start_time > $EXEC_TIME ));then
                    echo "Time has elapsed"
                    exit 0
                fi
            done
            unset cpu_list
            unset shuf_list
        done
    done
}

parse_arguments "$@"

if [ "$is_exec" = true ] && [ "$is_iter" = true ]; then
    echo "Both timeout and iterations cannot be used together"
    helpme
    exit 2
fi

idle_states=$(find /sys/devices/system/cpu/cpu*/cpuidle/state* -type d | cut -d'/' -f8 | sort -u | sed -e 's/^state//' | wc -l)

# Discover cpus and classify as cores
declare -a first_cpu_list=()
for ((cpu=0; cpu<NUM_CPUS; cpu++))
do
    if (( cpu % SMT == 0 )); then
        first_cpu_list+=($cpu)
    fi
done

rand_perm "${first_cpu_list[@]}"

## disable all idle states 
for ((state=0; state<idle_states; state++))
do
    for ((cpu=0; cpu<NUM_CPUS/SMT; cpu++))
    do
        for ((thread=${shuf_list[$cpu]}; thread<${shuf_list[$cpu]}+SMT; thread++))
        do
            # Disable the state for this thread
            # echo "Disable Idle for thread $thread"
            echo 1 > /sys/devices/system/cpu/cpu$thread/cpuidle/state$state/disable
            if (( $? != 0)); then
                echo "Disable idle for CPU ${idle_thread_list[$thread]} failed"
                exit 2
            fi
        done
    done
    unset shuf_list
    rand_perm "${first_cpu_list[@]}"
done

start_time=$(date +%s)
elapsed_time=$(date +%s)
## Run the tests
while true; do
    hotplug_test
    cpuidle_test
    if [ "$is_iter" = true ]; then
        exit 0
    fi
done