benchmark-zvols.sh

benchmark-zvols.sh

#!/bin/bash
#
# Use 10% of the pool free space to make multiple zvols, one zvol per CPU.
# Then write 5GB to each zvol in parallel, one writer per CPU.
# This test writes with non-O_DIRECT and O_DIRECT.
#
# Usage:
#   ./benchmarks-zvols.sh POOL

TESTPOOL=$1
if [ -z "$TESTPOOL" ] ; then
	echo "Usage:"
	echo "$0 POOLNAME"
	exit
fi

# Create one zvol per CPU
num_zvols=$(nproc)

ZFS=zfs
ZPOOL=zpool

# Uncomment these for development
# ZFS=`pwd`/cmd/zfs/zfs
# ZPOOL=`pwd`/cmd/zpool/zpool

# pool free space
pool_free=$(eval $ZPOOL list -pHo free $TESTPOOL)

# Use 10% of the pool's free space for zvols.  Each zvol get some slice of that.
zvol_size=$(( ($pool_free / 10) / $num_zvols))

# Write 5GB to each zvol
zvol_write_size=$((5 * 1024 * 1024 * 1024))

# $1: (optional) additional args for zfs create (like "-o volblocksize=1048576")
function create_zvols
{
	options="$1"
	echo "Creating $num_zvols zvols that are ${zvol_size}B each.  $options"
	for i in $(seq 1 $num_zvols) ; do
		eval $ZFS create -V $zvol_size $options $TESTPOOL/testvol$i
		eval $ZFS set compression=off $TESTPOOL/testvol$i
	done
}

function destroy_zvols
{
	for i in $(seq 1 $num_zvols) ; do
		eval $ZFS destroy $TESTPOOL/testvol$i
	done

}

# $1: "read" or "write"
# $2: (optional) additional arugments to pass to dd (like "oflag=direct")
function benchmark
{
	rw=$1
	bs=$((1024 * 1024))
	dd_options="$2"

	num_blocks=$(($zvol_write_size / $bs))
	echo -n "$rw ${num_blocks}MB each zvol"
	if [ -n "$dd_options" ] ; then
		echo -n " $dd_options"
	fi
	echo ""

	# Why are we writing to a script and then executing it (below)?
	#
	# If you spawn off the dd's and then do a `time wait` it works.
	# However, if you do the same thing, and do a
	# `time --format='blah' wait` it says "wait command not found".  The
	# workaround is to write the script and execute it so that you can
	# use `time --format...` (this can probably be improved).
	script=$(mktemp)
	echo "#!/bin/ksh -p" >> $script
	echo "for i in "$(seq 1 $num_zvols)" ; do" > $script
	if [ "$rw" == "read" ] ; then
		echo "  dd if=/dev/zvol/$TESTPOOL/testvol\$i of=/dev/null bs=$bs count=$num_blocks $dd_options 2>/dev/null & true" >> $script
	else
		echo "	dd if=/dev/zero of=/dev/zvol/$TESTPOOL/testvol\$i bs=$bs count=$num_blocks $dd_options 2>/dev/null & true" >> $script
	fi
	echo "done && wait" >> $script
	chmod +x $script

	out=$(/bin/time --format='%e %S %U' $script 2>&1)
	rm -f $script
	elapsed=$(echo $out  | cut -f 1 -d ' ')
	system=$(echo $out | cut -f 2 -d ' ')
	user=$(echo $out | cut -f 2 -d ' ')

	mb_sec=$(echo "($zvol_write_size * $num_zvols / 1048576 / $elapsed)" | bc)

	echo "$(printf 'MB/s\t\telapsed (sec)\tsystem (sec)\tuser (sec)\n')"
	echo "$(printf '%d\t\t'$elapsed'\t\t'$system'\t\t'$user'\n' $mb_sec)"
}

create_zvols "-o volblocksize=1048576"
benchmark "write"
benchmark "read"
destroy_zvols
echo ""
create_zvols "-o volblocksize=1048576"
benchmark "write" "oflag=direct"
benchmark "read" "iflag=direct"
destroy_zvols