nvmf

nvmf.sh

#!/bin/bash

CPUS="1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,0"
NIC="eth4"
LGB_NVMF_PORT=4420
SFC_NVMF_PORT=11345
WR_USR="rs"
#NIC="enp3s0f0"

trap "exit -1" TERM

die()
{
    echo -e $1 >&2
    kill -TERM $$
}

get_field_value()
{
    echo $(echo -n $1 | cut -f$2 -d$3)
}

extract_parameter_value()
{
    echo $(get_field_value $1 2 '=')
}

log_and_eval()
{
    echo "LOG($(basename $0)): $@"
    eval "$@"
    [ $? -ne 0 ] && {
	echo -e "\e[91m $@ \e[0m"
	die ""
    }
}

do_setup()
{
    DISABLED_NUMA_NODE=0

    #turn off MLNX IB daemons
    log_and_eval "/etc/init.d/openibd stop"
    log_and_eval "systemctl stop irqbalance"

    #modules to unload before disabling numa nodes
    for m in sfc ib_core mlx5_ib mlx5_core mlx_compat
    do
	:
	#log_and_eval "modprobe -r $m"
    done

    #enabled core nvme modules
    #for m in nvme_core nvme
    #do
    #	log_and_eval modprobe $m
    #done

    #turn off CPUs in disabled node
    #echo -e "\n\e[45m Bringing CPUs offline -- This is dellr630ap specific. Change if using another machine"
    #numa_node_cpus=$(numactl --hardware | grep "node $DISABLED_NUMA_NODE cpus" | cut -f2 -d:)
    #for c in $numa_node_cpus;
    #do
    #	log_and_eval "echo 0 > /sys/devices/system/cpu/cpu$c/online"
    #done
    #echo -e "\e[49m"

    #log_and_eval "/etc/init.d/openibd start"
    modprobe mlx5_core
    log_and_eval "rmmod ip_tables"
    $0 ip
}

setup_ip()
{
    nics=$(ip addr | grep enp.*\: -o | cut -f1 -d':')

    for n in $nics;
    do
	ip addr flush $n
    done
    log_and_eval ifconfig $NIC $(hostname -s)-l/24
}

#I like this better than getopts style args
extract_argval()
{
    key=$1
    val=""
    shift

    for k in $*
    do
	ckey=$(get_field_value $k 1 '=')
	if [ $ckey == $key ]; then
	    val=$(get_field_value $k 2 '=')
	fi
    done
    echo $val
}

get_man_argval()
{
    key=$1
    val=$(extract_argval $*)

    if [ -z $val ]; then
	die "mandatory argument \"$key\" missing"
    fi
    echo $val
}

kill_processes()
{
    for p in $*;
    do
	kill -9 $(ps axwu | grep $p | awk '{print $2}') 2>/dev/null
    done
}

init_stopped_cpu_logger()
{
    output=$1
    LOGTIME=5
    MPSTAT_CPU=0
    #MPSTAT=taskset -c $MPSTAT_RUN_CPU mpstat -P ALL 1 $LOGTIME | grep all
    #log_and_eval "(bash -c 'kill -SIGSTOP \$\$; $MPSTAT') > $output &"
    kill_processes mpstat
    taskset -c 0 bash -c "kill -STOP \$\$;mpstat -P ON 1 $LOGTIME 2>&1 | su -m -c \"cat > $output\" $WR_USR" &
}

kickoff_cpu_logger()
{
    logger_pid=$1
    kickoff_wait=$2
    taskset -c 0 bash -c "sleep $kickoff_wait; kill -cont $logger_pid" &
}

run_io_test()
{
    args="$*"
    extraparms=""

    FIO="/local/rip/fio/build/bin/fio"

    #get mandatory arguments
    test=$(get_man_argval test $args)
    ioengine=$(get_man_argval ioengine $args)
    iodepth=$(get_man_argval iodepth $args)
    nvmedev=$(get_man_argval nvmedev $args)
    blocksize=$(get_man_argval blocksize $args)
    numjobs=$(get_man_argval numjobs $args)
    logdir=$(get_man_argval logdir $args)
    target=$(get_man_argval target $args)
    verbose=$(extract_argval verbose $args)
    initiator=$(extract_argval initiator $args)
    runid=$(get_man_argval runid $args)

    [ ! -d $logdir ] && {
	die "logdir=$logdir does not exist"
    }

    [ $iodepth -lt $numjobs ] && die "iodepth should be >= numjobs"

    [ $((iodepth % numjobs)) -ne 0 ] && die "iodepth should be multiple of numjobs"

    if [ $ioengine == "pvsync2" ]; then
	extraparms="--hipri=1"
    fi

    minimal_cli="--minimal"
    iodepth_perjob=$((iodepth/numjobs))
    output_fio_basename="$logdir/test-$test.ioengine-$ioengine.iodepth-$iodepth.blocksize-$blocksize.numjobs-$numjobs.target-$target.runid-$runid"
    output_fio_file="$output_fio_basename.fio.txt"
    output_fio_tmp_file="/tmp/$$.fio.txt"

    [ ! -z $verbose ] && { output_fio_tmp_file="/dev/stdout"; minimal_cli=""; }

    kill_processes fio

    log_and_eval "ssh -f $target \"$0 __cpu_logger_remote $output_fio_basename.target.cpu.txt\" &"
    log_and_eval "ssh -f $initiator \"$0 __cpu_logger_remote $output_fio_basename.initiator.cpu.txt\" &"

    cpulist=$(echo $CPUS | cut -f1-$numjobs -d,)

    log_and_eval taskset -c $cpulist $FIO --name=test-$test --test=$test --filename=$nvmedev --ioengine=$ioengine --direct=1 --gtod_reduce=1 --bs=$blocksize \
		 --readwrite=$test --iodepth=$iodepth_perjob --runtime=15s --max-jobs=32 \
		 --numjobs=$numjobs --warnings-fatal --group_reporting=1 \
		 --clocksource=cpu --buffered=0 --ramp_time=1s \
		 --clat_percentiles=1 --disable_clat=0 \
		 --exitall_on_error --output=$output_fio_tmp_file $minimal_cli $extraparms

    [ -z $verbose ] && { su -m -c "cp $output_fio_tmp_file $output_fio_file" $WR_USR; rm "$output_fio_tmp_file"; }

    #log_and_eval "ssh -f $target \"/home/rs/mp/int-dist $NIC >$output_fio_basename.target.nic.intdist.txt \""
    #log_and_eval "ssh -f $target \"/home/rs/mp/int-dist nvme0 >$output_fio_basename.target.nvme0.intdist.txt \""
    #log_and_eval "/home/rs/mp/int-dist $NIC >$output_fio_basename.init.nic.intdist.txt"
}

get_file_param_set()
{
    logdir=$1
    key=$2

    set=$(ls -1 $logdir/*$key* | egrep -o "$key-[a-z0-9_]+" | cut -f2 -d- | sort -n | uniq)
    echo $set
}

get_fio_cval()
{
    fn=$1
    colname=$2
    output=""

    case $colname in
	'read-bandwidth')
	    output=$(cat $fn | cut -f 7 -d ';')
	    ;;

	'write-bandwidth')
	    output=$(cat $fn | cut -f 48 -d ';')
	    ;;

	'read-minclat')
	    output=$(cat $fn | cut -f 14 -d ';')
	    ;;

	'read-meanclat')
	    output=$(cat $fn | cut -f 16 -d ';')
	    ;;

	'read-maxclat')
	    output=$(cat $fn | cut -f 15 -d ';')
	    ;;

	'read-stdevclat')
	    output=$(cat $fn | cut -f 17 -d ';')
	    ;;

	'write-minclat')
	    output=$(cat $fn | cut -f 55 -d ';')
	    ;;

	'write-meanclat')
	    output=$(cat $fn | cut -f 57 -d ';')
	    ;;

	'write-maxclat')
	    output=$(cat $fn | cut -f 56 -d ';')
	    ;;

	'write-stdevclat')
	    output=$(cat $fn | cut -f 58 -d ';')
	    ;;

	'read-iops')
	    output=$(cat $fn | cut -f 8 -d ';')
	    ;;

	'write-iops')
	    output=$(cat $fn | cut -f 49 -d ';')
	    ;;
	'write-clat-95th-percentile')
	    output=$(cat $fn | cut -f 70 -d ';' | cut -f 2 -d=)
	    ;;
	'read-clat-95th-percentile')
	    output=$(cat $fn | cut -f 29 -d ';' | cut -f 2 -d=)
	    ;;
	*)
	    die "get_fio_cval:unknown column: $colname"
	    ;;
    esac

    [ $? -ne 0 ] && output="?"


    output=$(printf '%0.2f' $output)
    echo $output
}

get_cpu_cval()
{
    fn=$1
    colname=$2
    output="-"

    case $colname in
	'usr')
	    output=$(cat $fn | grep 'Average' | grep 'all' | awk '{print $3}')
	    ;;

	'sys')
	    output=$(cat $fn | grep 'Average' | grep 'all' | awk '{print $5}')
	    ;;
	'irq')
	    irqtime=$(cat $fn | grep 'Average' | grep 'all' | awk '{print $7}')
	    softirqtime=$(cat $fn | grep 'Average' | grep 'all' | awk '{print $8}')
	    output=$(echo "$irqtime + $softirqtime" | bc)
	    ;;

	    *)
		die "get_cpu_res_col_val: unknown column: $colname"
	;;
esac

    [ $? -ne 0 ] && output="!"
    output=$(printf '%0.2f' $output)
    echo $output
}

gen_new_row()
{
    logdir=$1
    target=$2
    test=$3
    jobs=$4
    iodepth=$5
    blocksize=$6
    runid=$7

    declare -A chdrs=(
	['blocksize']='-'

	['fio_threads']='-'

	['init_user_cpu']='-'
	['init_sys_cpu']='-'
	['init_irq_cpu']='-'

	['target_user_cpu']='-'
	['target_sys_cpu']='-'
	['target_irq_cpu']='-'

	['init_read_bw']='-'
	['init_write_bw']='-'

	['init_read_iops']='-'
	['init_write_iops']='-'

	['init_read_minclat']='-'
	['init_read_maxclat']='-'
	['init_read_meanclat']='-'
	['init_read_stdevclat']='-'
	['init_read_9950_clat']='-'

	['init_write_minclat']='-'
	['init_write_maxclat']='-'
	['init_write_meanclat']='-'
	['init_write_stdevclat']='-'
	['init_write_9950_clat']='-'
    )

    if [ $logdir == "print-column-headers" ]; then
	for k in "${!chdrs[@]}";
	do
	    echo "$k"
	done

	return
    fi

    tgt_fio_fn="$logdir/test-$test.ioengine-*.iodepth-$iodepth.blocksize-$blocksize.numjobs-$jobs.target-$target.runid-$runid.fio.txt"
    init_cpu_fn="$logdir/test-$test.ioengine-*.iodepth-$iodepth.blocksize-$blocksize.numjobs-$jobs.target-$target.runid-$runid.initiator.cpu.txt"
    tgt_cpu_fn="$logdir/test-$test.ioengine-*.iodepth-$iodepth.blocksize-$blocksize.numjobs-$jobs.target-$target.runid-$runid.target.cpu.txt"

    if ! test -a $tgt_fio_fn; then
	return
    fi

    chdrs['blocksize']=$blocksize
    chdrs['fio_threads']=$jobs

    chdrs['init_user_cpu']=$(get_cpu_cval $init_cpu_fn 'usr')
    chdrs['init_sys_cpu']=$(get_cpu_cval $init_cpu_fn 'sys')
    chdrs['init_irq_cpu']=$(get_cpu_cval $init_cpu_fn 'irq')

    chdrs['target_user_cpu']=$(get_cpu_cval $tgt_cpu_fn 'usr')
    chdrs['target_sys_cpu']=$(get_cpu_cval $tgt_cpu_fn 'sys')
    chdrs['target_irq_cpu']=$(get_cpu_cval $tgt_cpu_fn 'irq')

    chdrs['init_read_bw']=$(get_fio_cval $tgt_fio_fn "read-bandwidth")
    chdrs['init_write_bw']=$(get_fio_cval $tgt_fio_fn "write-bandwidth")

    chdrs['init_read_iops']=$(get_fio_cval $tgt_fio_fn "read-iops")
    chdrs['init_write_iops']=$(get_fio_cval $tgt_fio_fn "write-iops")

    chdrs['init_read_minclat']=$(get_fio_cval $tgt_fio_fn "read-minclat")
    chdrs['init_read_maxclat']=$(get_fio_cval $tgt_fio_fn "read-maxclat")
    chdrs['init_read_meanclat']=$(get_fio_cval $tgt_fio_fn "read-meanclat")
    chdrs['init_read_stdevclat']=$(get_fio_cval $tgt_fio_fn "read-stdevclat")
    chdrs['init_read_9950_clat']=$(get_fio_cval $tgt_fio_fn "read-clat-95th-percentile")

    chdrs['init_write_minclat']=$(get_fio_cval $tgt_fio_fn "write-minclat")
    chdrs['init_write_maxclat']=$(get_fio_cval $tgt_fio_fn "write-maxclat")
    chdrs['init_write_meanclat']=$(get_fio_cval $tgt_fio_fn "write-meanclat")
    chdrs['init_write_stdevclat']=$(get_fio_cval $tgt_fio_fn "write-stdevclat")
    chdrs['init_write_9950_clat']=$(get_fio_cval $tgt_fio_fn "write-clat-95th-percentile")

    for k in "${!chdrs[@]}";
    do
	echo "${chdrs[$k]}"
    done
}


do_calc_test_result_stats()
{
    tests=$(get_file_param_set $logdir "test")
    blocksizes=$(get_file_param_set $logdir "blocksize")
    jobs=$(get_file_param_set $logdir "numjobs")
    targets=$(get_file_param_set $logdir "target")
    iodepths=$(get_file_param_set $logdir "iodepth")
    runs=$(get_file_param_set $logdir "runid")

    column_headers=$(gen_new_row "print-column-headers")
    echo -e $column_headers"\n"

    for g in $targets;
    do
	for t in $tests;
	do
	    for j in $jobs;
	    do
		for i in $iodepths;
		do
		    # constraint of iodepth >= nr_jobs means this is not possible, so ignore
		    [ $i -lt $j ] && continue

		    for r in $runs;
		    do
			## At this point we've enough context to gen one set of results
			echo "#target:$g,test:$t,jobs:$j,iodepth:$i,run:$r"
			for b in $blocksizes;
			do
			    result_line=$(gen_new_row $logdir $g $t $j $i $b $r)
			    [ ! -z "$result_line" ] && echo $result_line
			done
		    done
		    echo -e '\n'
		done
	    done
	done
    done
}

# coarse check to verify if we may be artificially limiting performance due to running out of CPU
do_sanity_check_cpu_files()
{
    logdir=$1

    for f in $logdir/*.cpu*;
    do
	#$6 = iowait  - which we ignore
	if awk '{print $3,$4,$5,$7,$8,$9,$10,$11}' $f | egrep '9[0-9]\.' 1>/dev/null ; then
	    echo "suspiciously high value in CPU file $f"
	fi
    done
}

do_calc_results()
{
    args="$*"

    #not really used at the moment
    test_type=$(get_man_argval test_type $args)

    logdir=$(get_man_argval logdir $args)

    do_sanity_check_cpu_files $logdir

    case $test_type in

	"io-bw-test" | "io-lat-test" | "io-randio-test")
	    do_calc_test_result_stats $args
	    ;;
	*)
	    die "unknown test_type=$test_type"
	    ;;

    esac
}

do_setup_nvmf_target()
{
    transport=$1
    dev=$(extract_argval dev $args)
    ip=$(host -i $(hostname -s)-l | egrep -o '[0-9.]+$')
    impl=$(extract_argval impl $args)
    dev=${dev:-"/dev/nvme0n1"}

    modprobe nvme_fabrics
    modprobe nvmet_rdma
    modprobe nvmet
    modprobe null_blk submit_queues=32 home_node=1 queue_mode=2 gb=200 hw_queue_depth=1023

    if [[ ! -a $dev ]]; then
	die "dev=$dev does not exist"
    fi

    if [ $impl == "sfc" ]; then
	module="/root/nvmf-tcp/drivers/nvme/target/nvmet-tcp.ko"
	portnum="11345"
    else
	module="/local/rip/linux-nvme-over-fabrics/drivers/nvme/host/nvme-tcp.ko"
	portnum="4420"
    fi

    insmod $module

    #Create "data" namespace
    mkdir /sys/kernel/config/nvmet/subsystems/data
    mkdir /sys/kernel/config/nvmet/subsystems/data/namespaces/1

    #Attach $nvmedev to it
    echo -n $dev > /sys/kernel/config/nvmet/subsystems/data/namespaces/1/device_path

    #Allow connections from any initiator
    echo 1 > /sys/kernel/config/nvmet/subsystems/data/attr_allow_any_host
    echo 1 > /sys/kernel/config/nvmet/subsystems/data/namespaces/1/enable

    # create the nvmet port
    mkdir /sys/kernel/config/nvmet/ports/1
    echo "ipv4" > /sys/kernel/config/nvmet/ports/1/addr_adrfam
    echo "$transport" > /sys/kernel/config/nvmet/ports/1/addr_trtype

    # port and listening IP address
    #echo "11345" > /sys/kernel/config/nvmet/ports/1/addr_trsvcid
    echo "$portnum" > /sys/kernel/config/nvmet/ports/1/addr_trsvcid
    echo "$ip" > /sys/kernel/config/nvmet/ports/1/addr_traddr

    #make the data target available over port 1
    ln -s /sys/kernel/config/nvmet/subsystems/data /sys/kernel/config/nvmet/ports/1/subsystems/data
}

do_setup_nvmf_initiator()
{
    modprobe nvme
    modprobe nvme_fabrics
    insmod /home/rs/mp/nvmf/nvmf-tcp/drivers/nvme/host/nvme-tcp.ko

}

do_unsetup_nvmf()
{
    /local/rip/nvme-cli/nvme disconnect -n data 2>/dev/null

    rm -f /sys/kernel/config/nvmet/ports/1/subsystems/data
    rmdir /sys/kernel/config/nvmet/subsystems/data/namespaces/1
    rmdir /sys/kernel/config/nvmet/subsystems/data
    rmdir /sys/kernel/config/nvmet/ports/1
    rmmod nvmet_rdma
    rmmod nvmet_tcp
    rmmod nvmet
    rmmod nvme_tcp
    rmmod nvme_fabrics
    #rmmod nvme
}

do_setup_interrupt_affinity()
{
    args="$*"
    target=$(get_man_argval target $args)
    log_and_eval "ssh $target /usr/sbin/ethtool -C $NIC rx-frames 0 tx-usecs 0 tx-frames 0; /bin/true"
    log_and_eval "ssh $target /home/rs/mp/oorun --stack=ke --dev=$NIC --interrupt_config=$NIC-*:stripe-node1 /bin/true"

}

do_setup_nvmf()
{
    args="$*"

    transport=$(get_man_argval transport $args)
    role=$(get_man_argval role $args)
    impl=$(get_man_argval impl $args)

    if [ $transport != "tcp" ] && [ $transport != "rdma" ]; then
	die "unknown transport type"
    fi

    if [ $impl != "sfc" ] && [ $impl != "lgb" ]; then
	die "impl should be one of [sfc|lgb]"
    fi

    if [ $role == "init" ]; then
	do_setup_nvmf_initiator

    elif [ $role == "targ" ]; then
	do_setup_nvmf_target $transport $impl
    else
	die "unknown role value=$role"
    fi
}

do_compare_results()
{
    args="$*"
    results=$(get_man_argval results $args)
    column=$(get_man_argval columns $args)

    test=$(get_man_argval test $args)
    jobs=$(get_man_argval jobs $args)
    iodepth=$(get_man_argval iodepth $args)

    results=$(echo $results | sed -e s/,/\ /g)
    #columns=$(echo $columns | sed -e s/,/\ /g)

    ## This is brittle.  Ideally we should pick the column position from a header in the
    ## output file, not rely on a fixed field representation.
    ch=" \
 	 init_user_cpu \
	 init_sys_cpu \
	 init_irq_cpu \
	 target_user_cpu \
	 target_sys_cpu \
	 target_irq_cpu \
	 init_read_bw \
	 init_write_bw \
	 init_read_iops \
	 init_write_iops \
	 init_read_minclat \
	 init_read_maxclat \
	 init_read_meanclat \
	 init_read_stdevclat \
	 init_read_9950_clat \
	 init_write_minclat \
	 init_write_maxclat \
	 init_write_meanclat \
	 init_write_stdevclat \
	 init_write_9950_clat \
         all_cpu"

    ### XXXX HACK -- sum all cpu
    if [ $column == "all_cpu" ]; then

	for r in $results;
	do
	    echo "-------------- $r $column [-1] ----------------"
	    rfile="/tmp/rfile.tmp"
	    $0 compare-results test=$test jobs=$jobs iodepth=$iodepth results=$r columns="init_user_cpu" > $rfile
	    $0 compare-results test=$test jobs=$jobs iodepth=$iodepth results=$r columns="init_sys_cpu" >> $rfile
	    $0 compare-results test=$test jobs=$jobs iodepth=$iodepth results=$r columns="init_irq_cpu" >> $rfile
	    $0 compare-results test=$test jobs=$jobs iodepth=$iodepth results=$r columns="target_user_cpu" >> $rfile
	    $0 compare-results test=$test jobs=$jobs iodepth=$iodepth results=$r columns="target_sys_cpu" >> $rfile
	    $0 compare-results test=$test jobs=$jobs iodepth=$iodepth results=$r columns="target_irq_cpu" >> $rfile

	    recs=$(cat $rfile | grep -v ^- | cut -f1 -d' ' | sort -n | uniq)
	    for c in $recs;
	    do
		output=$(grep $c $rfile | cut -f2 -d' ' | paste -sd + | bc)
		output=$(printf '%0.2f' $output)
		echo $c $sum $output
	    done
	done
	return 0
    fi
    ### XXXX HACK -- sum all cpu


    colnum=$(echo $ch | tr ' ' '\n' | grep -n $column)
    [ $? -ne 0 ] && die "no such column: $column"
    colnum=$(echo $colnum | cut -f1 -d:)
    colnum=$((colnum + 1))

    for r in $results;
	     do
		 echo "------------ $r - $column [$colnum]----------------"

		 end=$(grep -A 100 "test:$test,jobs:$jobs,iodepth:$iodepth$" $r | grep -n '$^')

		 [ $? -ne 0 ] && die "unable to find signature"

		 end=$(echo $end | cut -f1 -d:)
		 end=$((end - 2))
		 output=$(grep -A $end "test:$test,jobs:$jobs,iodepth:$iodepth$" $r | cut -f1,$colnum -d' ')

		 [ $? -ne 0 ] && die "unable to extract column"

		 echo -e "$output " | grep -v ^#
    done

    exit 0


}

do_print_calced_result()
{
    args="$*"
    resultfile=$(get_man_argval resultfile $args)
    columns=$(get_man_argval columns $args)
    columns=$(echo $columns | sed -e s/,/\ /g)
    awkcols=""

    [ ! -f $resultfile ] && die "result file $resultfile not found"

    colkey=$(head -n 1 $resultfile | sed -e 's/\ /\n/g' | cat -n)

    for c in $columns;
    do
	cnum=$(echo -e "$colkey" | grep "$c" | tr -d ' ' | cut -f1)
	[ -z $cnum ] && die "column $c not present in result file"
	awkcols+="\$$cnum,"
    done

    awkcols=$(echo $awkcols | sed -e 's/.$//')

    cat $resultfile | awk "{print $awkcols}" | grep -v '^\ *$' | sed -e 's/\ /, /g'

}

do_clear_ntuple_rules()
{
    nr_rules=$(($(ethtool --show-nfc $NIC | grep "Total " | cut -f2 -d' ' )))

    if [ $nr_rules -eq 0 ]; then
	return
    fi

    for s in $(seq 0 $nr_rules);
    do
	ethtool --config-ntuple $NIC delete $s
    done

}

do_setup_ntuple_rules()
{
    args="$*"
    srcport=$(extract_argval match-srcport $args)
    dstport=$(extract_argval match-dstport $args)

    ruleindex=$((0))
    rings=$(($(ethtool --show-nfc $NIC | egrep '[0-9]+\ RX' | cut -f1 -d' ')))

    [ -z $srcport ] && [ -z $dstport ] && \
	die "match-dstport or match-srcport must be set"

    do_clear_ntuple_rules

    if [ ! -z "$dstport" ]; then
	port_match="dport"
	sscol=4
	portnum_to_match=$(get_field_value $dstport 2 ':')
    else
	port_match="sport"
	sscol=5
	portnum_to_match=$(get_field_value $srcport 2 ':')
    fi

    conns=$(ss -o state established "( $port_match = :$portnum_to_match )" | awk "{print \$$sscol}" | tail -n +2)

    for c in $conns;
    do

	if [ $port_match == "dport" ]; then
	    saddr=$(get_field_value $dstport 1 ':')
	    sport=$(get_field_value $dstport 2 ':')
	    daddr=$(get_field_value $c 1 ':')
	    dport=$(get_field_value $c 2 ':')
	else
	    daddr=$(get_field_value $srcport 1 ':')
	    dport=$(get_field_value $srcport 2 ':')
	    saddr=$(get_field_value $c 1 ':')
	    sport=$(get_field_value $c 2 ':')
	fi

	log_and_eval "ethtool --config-ntuple $NIC flow-type tcp4 src-ip $saddr src-port $sport dst-ip $daddr dst-port $dport loc $ruleindex action $(($ruleindex % $rings))"

	ruleindex=$((ruleindex + 1))
    done
}

case $1 in
    "setup")
	shift
	do_setup $*
	;;
    "ip")
	shift
	setup_ip $*
	;;
    "io-test")
	shift
	args="$*"
	init=$(get_man_argval initiator $args)
	tgt=$(get_man_argval target $args)
	nrruns=$(get_man_argval nrruns $args)

	do_setup_interrupt_affinity target=$init
	do_setup_interrupt_affinity target=$tgt

	#setup ntuple rules for SFC driver
	log_and_eval "ssh $init $0 __setup_ntuple_rules match-dstport=$(getent hosts $tgt | cut -f1 -d' '):$SFC_NVMF_PORT"
	log_and_eval "ssh $tgt $0 __setup_ntuple_rules match-srcport=$(getent hosts $tgt | cut -f1 -d' '):$SFC_NVMF_PORT"

	#setup ntuple rules for LGB driver
	log_and_eval "ssh $init $0 __setup_ntuple_rules match-dstport=$(getent hosts $tgt | cut -f1 -d' '):$LGB_NVMF_PORT"
	log_and_eval "ssh $tgt $0 __setup_ntuple_rules match-srcport=$(getent hosts $tgt | cut -f1 -d' '):$LGB_NVMF_PORT"

	for r in $(seq 1 $nrruns);
	do
	    run_io_test $* runid=$r
	done
	;;
    "__cpu_logger_remote")
	shift
	outfile=$1
	shift
	init_stopped_cpu_logger $outfile
	kickoff_cpu_logger $! 5
	;;
    "calc-results")
	shift
	do_calc_results $*
	;;
    "setup-nvmf")
	shift
	do_setup_nvmf $*
	;;
    "unsetup-nvmf")
	do_unsetup_nvmf $*
	;;
    "compare-results")
	do_compare_results $*
	;;
    "print-calced-result")
	do_print_calced_result $*
	;;
    "__setup_ntuple_rules")
	do_setup_ntuple_rules $*
	;;
    *)
	die "$(basename $0): unrecognised or no action"
	;;
esac

exit 0