megacli raid adapter and lvm tests

README.md

megacli testing with fio

This repo is here to help do some benchmarking of raid adapter performance and lvm raid/raid0 as well.

To get started:

• boot a bionic livecd.
• git clone https://gist.github.com/smoser/704f1a38c9ffafece5f2fc457dd8cdd0 bench
• cd bench
• run sudo ./run

I suggest running in screen or tmux (./setup will install those).

It should install dependencies and put output in results/.

Tests

The tests that are run are run by ./run-tests.

FAQ

• Why didn't you use a .fio file? Well... I tried initially to have a fio that defined multiple tests, but unlink=1 didn't seem to delete the files after tests inside job file. Probably there was just something I was doing wrong.

collect-info

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh || exit 1

[ "$(id -u)" = "0" ] || fail "not root"

# https://wiki.mikejung.biz/LSI#How_to_show_all_LSI_RAID_info_with_MegaCLI
results_d="$1"
base="${results_d}/system"
mkdir -p "${results_d}" || fail "failed mkdir $results_d"

{
section MegaCLI info
runi megacli -LDInfo -Lall -aAll
runi megacli -AdpAllInfo -a0
runi megacli -PDList -a0

section lvm info
runi pvs -o pv_all
runi lvs -o lv_all

section dmi
runi sh -c '( cd /sys/class/dmi/id/ && grep -r . . )'

section other info
runi blockdev --report /dev/[sv]d?
runi udevadm info /dev/[sv]d?
runi df
} > "${base}-info.txt" 2>&1

lshw -json >"${base}-lshw.json"

common.sh

VERBOSITY=${VERBOSITY:-2}
MPBASE="${MPBASE:-/test-mounts}"
if [ "${_PATH_SET:-0}" = "0" ]; then
    PATH=$HOME/bin:$(dirname ${BASH_SOURCE}):/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
    _PATH_SET=1
fi

assert_root() { [ "$(id -u)" = "0" ] || fail "not root"; }
error() { echo "$@" 1>&2; }
msg() {
    [ "$VERBOSITY" -ge "$1" ] || return
    shift
    echo "$@" 1>&2
}
fail() { msg 0 "[FATAL]" "$@"; exit 1; }
warn() { msg 0 "[WARN]" "$@"; }
info() { msg 1 "[INFO]" "$@"; }
debug() { msg 2 "[DEBUG]" "$@"; }
trace() { msg 3 "[TRACE]" "$@"; }
section() { msg 0 "|=========" "$@" "==========|"; }
hascmd() { command -v "$1" >/dev/null 2>&1; }

wait_for_lvm_sync() {
    local out="" missing="" sstart="$SECONDS"
    while :; do
        out=$(lvs "--separator=|" --options=name,copy_percent --no-headings) || fail "Failed to execute lvs"
        missing=$(echo "$out" |
            awk '-F|' '
                $2 != "" && $2 != "100.00" {
                    gsub(/ /, ""); wait=wait " " $1 "/" $2; };
                END { printf("%s\n", wait); }')
        [ -z "$missing" ] && {
            info "waited $((SECONDS-sstart))s for lvm sync"
            return
        }
        debug "Waiting on lv sync: $missing"
        sleep 1
    done
}

shellquote() {
    # quote the arguments as appropriate for shell.
    local x="" wrap="" tickset=$'"!$' quoteset=" '" tick="'" cmd=""
    for x in "$@"; do
        if [ "${x#*[${tickset}]}" != "$x" ]; then
            wrap="'"
            x=${x//$tick/$tick\\$tick$tick}
        elif [ "${x#*[$quoteset]}" != "$x" ]; then
            wrap='"'
        else
            wrap=""
        fi
        cmd="${cmd} $wrap$x$wrap"
    done
    _RET="${cmd# }"
}

getmissing() {
    local cmd="" pkg=""
    for pkg in "$@"; do
        cmd="$pkg"
        case "$pkg" in
            *:*) cmd=${pkg%:*}; pkg="${pkg#*:}";;
        esac
        hascmd "$cmd" || echo "$pkg"
    done
}

run(){
    # run(level=[warn,info,debug,trace], [quiet], cmd...)
    local loglvl="$1" rc="" m=""
    shift
    [ "${1#-m}" = "$1" ] || { m="${1#-m}"; shift; }
    shellquote "$@"
    $loglvl ${m:+"$m:"} "execute:" "$_RET"
    "$@"
}

runi() { run info "$@"; }
runq() {
    # runq: like run, but only show command output on error.
    local loglvl="$1" out="" rc=""
    shift
    [ "${1#-m}" = "$1" ] || { m="${1#-m}"; shift; }
    shellquote "$@"
    $loglvl ${m:+"$m:"} "execute:" "$_RET"
    out=$("$@" 2>&1) && return 0
    rc=$?
    warn "cmd [$*] failed with exit code '$rc': $out"
    return $rc
}

domount() {
    local dev="$1" name="$2"
    mkdir -p "$MPBASE/$name"
    [ ! -e "$dev" ] && [ -e "/dev/$dev" ] && dev="/dev/$dev"
    run info mount "$dev" "$MPBASE/$name"
}

domkfs() {
    local dev="$1" name="$2"
    runq info mkfs.ext4 "-L$name" "$dev"
}

fsmount() {
    local dev="$1" name="$2"
    [ ! -e "$dev" ] && [ -e "/dev/$dev" ] && dev="/dev/$dev"
    [ -e "$dev" ] || { warn "$dev does not exist"; return 1; }
    domkfs "$dev" "$name" || return
    domount "$dev" "$name"
}

get-disks

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh || exit 1

# get devices > 500GiB
gb=$((1024*1024*1024))
min=$((${1:-500}*gb))
devs=$(for d in /dev/[vs]d?; do [ -b "$d" ] && echo "$d"; done)
out=$(set -f; sudo blockdev --report $devs) ||
    fail "failed to run blockdev on $devs"
found=$(echo "$out" |
   awk '$6 != "Size" && $6 > min { print $7 }' min=$min)
[ -n "$found" ] || fail "no devs > $((min/gb))GiB found. tried $devs".
echo $found

run

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh || exit 1
myd=$(dirname "$BASH_SOURCE") && myd=$(cd "$myd" && pwd)

assert_root
PATH=$myd:$PATH

results_d=${1:-./results}
name=${2}
[ -n "$name" ] || fail "Must give name."

mkdir "$results_d" || fail "failed to create $results_d"
echo "$name" > "$results_d/name.txt"

devs=$(get-disks 500)

setup || fail "failed setup"
info "running.  results in $results_d: $name"
info "wiping to ${results_d}/wipe.log"
wipe > "${results_d}/wipe.log" 2>&1 || fail "failed wipe"

info "settings up lvs/partitions on $devs"
setup-lvs $devs > "${results_d}/setup-lvs.log" 2>&1 || fail
info "writing info to ${results_d}/system-info"
collect-info "$results_d/" || fail
info "running tests output to ${results_d}/run-tests.log"
run-tests "${results_d}/" 2>&1 |
    tee "${results_d}/run-tests.log"
info "done [$SECONDS]"

run-tests

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh || exit 1
mydir=$(dirname "$BASH_SOURCE") || fail

sizes="4k 16k 32k"
case "${ALL:-unset}" in
    0|""|unset|false) all=false;;
    *) all=true;;
esac

if [ "$1" = "-a" -o "$1" = "--all" ]; then
    all=true
    shift
fi
[ $# -ge 1 ] || fail "must give results directory."

results_d="$1"
mkdir -p "$results_d" || fail "failed to create $results_d"
shift
results_d=$( cd "$results_d" && pwd ) ||
    fail "failed to get full path to $results_d"

common=(
    --direct=1
    --numjobs=8
    --time_based=1
    --runtime=600
    --ioengine=libaio
    --unlink=1
    --size=2G
    --group_reporting
    #--iodepth=16
)


dorun() {
    local dir="$1" name="$2" log="$3" ss="$SECONDS"
    shift 3
    [ -f "$log" ] && {
        info "skipping $name per log $log existed"
        return
    }
    info "testing name=$name dir=$dir log=$log : $*"
    ( cd "$dir" && runi fio --name="$name" "$@" ) > "$log" 2>&1
    info "finished $? in $((SECONDS-ss))s"
    rm -f "$dir/$name".*
}

for d in $MPBASE/*; do
    [ -d "$d" ] || continue
    dname=${d##*/}
    case "$dname" in
        *-ext4) dtype="ext4";;
        *) dtype=$dname;;
    esac
    if ! $all; then
        case " $dtypes " in
            *\ $dtype\ *)
                info "already ran dtype=$dtype, skipping $dname"
                continue;;
        esac
    fi
    dtypes="$dtypes $dtype"
    rbase="${results_d}/test-${dname}"
    for bs in $sizes; do
        dorun "$d" "read-$bs" "${rbase}.read-$bs.log" \
            "${common[@]}" "--rw=read" "--bs=$bs"
    done
    for bs in $sizes; do
        dorun "$d" "write-$bs" "${rbase}.write-$bs.log" \
            "${common[@]}" "--rw=write" "--bs=$bs"
    done
done
info "finished in ${SECONDS}s"

setup

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh || exit 1
myd=$(dirname "$BASH_SOURCE") && myd=$(cd "$myd" && pwd)

assert_root

cat > $HOME/.screenrc <<EOF
hardstatus alwayslastline "%-Lw%{= BW}%50>%n%f* %t%{-}%+Lw%< %= %H"
defscrollback 4096
EOF

suser=${SUDO_USER}
if [ -n "$suser" ]; then
    ( umask 226 &&
      echo "$suser ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/$suser-user" ) ||
      fail "failed to give $suser sudo"
fi


if [ "${#pkgs[@]}" != "0" ]; then
    pkgs=( $(getmissing fio screen tmux) )
    runi sudo apt-get --quiet update || fail
    info "installing ${pkgs[*]}"
    runi sudo apt-get --quiet install --assume-yes "${pkgs[@]}" || fail
else
    info "already had all packages."
fi

setup-lvs

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh || exit 1

genparted() {
    # genparted(disk, [name size other-info, ...)
    # Generate a parted command with just names and sizes.
    #  sets the varibable 'cmd' and associative array ptbyname
    #    ptbyname[name]=<partition-number>
    #  other-info can contain NUM which will be replaced with
    #  the current partition number.
    local disk="$1" size name extra start cstart args arg="" ptnum=1 fs=""
    shift
    args=( "$@" )
    cmd=(
        parted --script --align=optimal "$disk"
        mklabel gpt
        unit MiB
    )
    start=3
    local i=0
    while [ $i -lt ${#args[@]} ]; do
        name=${args[$i]}
        size=${args[$i+1]}
        extra=${args[$i+2]}
        i=$((i+3))
        case "$size" in
            *M) size=$(((${size%M}*1000)/1024));;
            *MiB) size=${size%MiB};;
            *G) size=$(((${size%G}*1000*1000)/1024));;
            *GB) size=$(((${size%GB}*1000*1000)/1024));;
            *GiB) size=$((${size%GiB}*1024));;
        esac

        [ "$extra" = "-" ] && extra=""
        set -- $extra
        fs=""
        cstart=$start
        while [ $# -ge 1 ]; do
            case "$1" in
                fs=*) fs=${1#fs=};;
                # allows out of order partitions
                start=*) cstart=${1#start=};;
                *) break;;
            esac
            shift
        done
        extra="${*//NUM/$ptnum}"

        cmd=( "${cmd[@]}"
              mkpart primary $fs $cstart $(($cstart+$size))
              name $ptnum $name $extra )
        if [ $start = $cstart ]; then
            start=$((start+size+1))
        fi
        ptbyname[$name]="$ptnum"
        ptnum=$((ptnum+1))
    done
    cmd=( "${cmd[@]}" )
}

_lvcreate() {
    run info \
        lvm lvcreate --nosync --ignoremonitoring --yes --activate=y \
            --setactivationskip=n "$@"
    wait_for_lvm_sync
    udevadm settle
}

main() {
    if [ $# -eq 0 ]; then
        fail "must provide disks"
    fi
    local disk="" fspart lvpart plpath="/dev/disk/by-partlabel"
    local num_disks=$#
    for disk in "$@"; do
        name=${disk##*/}
        run debug genparted $disk \
            $name-bare 20GiB - \
            $name-ext4 20GiB - \
            $name-lvm  100GiB "set NUM LVM on"
        run info "${cmd[@]}" print
        udevadm settle
    done
    for fspart in $plpath/*-ext4; do
        [ -b "$fspart" ] || fail "What? no $fspart block dev"
        fsmount "$fspart" "${fspart##*/}" || fail
    done

    if [ "${SKIP_LVM_RAID:-0}" != "0" ]; then
        info "Skipping lvm raid tests. (SKIP_LVM_RAID=$SKIP_LVM_RAID)"
    else
        info "Setting up lvm raid mounts"
        for lvpart in $plpath/*-lvm; do
            [ -b "$fspart" ] || fail "What? no $fspart block dev"
            run info lvm pvcreate "$lvpart" || fail
        done

        vg="vg0"
        run info lvm vgcreate $vg $plpath/*-lvm || fail

        _lvcreate --type=raid1 --size=20G --name="raid1" "$vg" ||
            fail "failed create raid1"
        fsmount "$vg/raid1" "raid1" || fail

        _lvcreate --type=raid0 --stripes=2 --size=20G --name="raid0-2" $vg ||
            fail "failed lvcreate stripes2"
        fsmount "$vg/raid0-2" "raid0-2" || fail

        if [ $num_disks -ge 4 ]; then
            _lvcreate --type=raid0 --stripes=4 --size=20G --name="raid0-4" $vg ||
                fail "failed lvcreate stripes4"
            fsmount "$vg/raid0-4" "raid0-4" || fail
        else
            info "Not creating 4 stripes, as only $num_disks disks"
        fi
    fi

    wait_for_lvm_sync
}

assert_root

main "$@"

wipe

#!/bin/bash
. $(dirname "$BASH_SOURCE")/common.sh

quiet() {
    local out="" rc=""
    out=$("$@" 2>&1)
    rc=$?
    [ $rc -eq 0 ] && return
    error "[$rc]: $*"
    error "$out"
    return $rc
}

wipe_blockdev() {
    # wipe_blockdev(device, num). zero the first and last num MiB of device.
    local dev="$1" zero_num="$2" sectors="" size_mb="" count="" leftover=""
    [ -b "$dev" ] || { error "$dev: not a block device"; return 1; }
    sectors=$(blockdev --getsz "${DEVPART}")
    size_mb=$((sectors/2048))
    leftover=$((sectors-(size_mb*2048)))
    [ ${size_mb} -ge $zero_num ] && count=$zero_num || count=$size_mb
    info "zero-ing $dev first ${count}M" \
       "(sectors=$sectors size=${size_mb}M num=$zero_num leftover=$leftover)"
    quiet dd if=/dev/zero of="$dev" bs=1M count=$count || {
        error "Failed to zero front of $dev sectors=$sectors" \
            "size_mb=$size_mb count=$count"
        return 1
    }
    if [ $size_mb -gt $zero_num ]; then
        info "zero-ing $dev last ${zero_num}M"
        local seek=$((size_mb-zero_num))
        quiet dd if=/dev/zero of="$dev" bs=1M count=$zero_num seek=$seek || {
            error "Failed to zero end of $dev sectors=$sectors" \
                "size_mb=$size_mb count=$count"
            return 1
        }
        # if size was not even MB (likely) then we have to get to the end.
        if [ $leftover -ne 0 ]; then
            info "Finishing zero of $dev for $leftover sectors"
            quiet dd if=/dev/zero "of=$dev" bs=512 count=$leftover \
                seek=$((sectors-leftover)) || {
                error "Failed to wipe remaining $leftover sectors on $dev"
                return 1
            }
        fi
    fi
}

wipe_lvs() {
    info "Clearing all LVs and partitions off of all non-removable disks"
    info "Clearing all known LVs"
    for PV in $(lvm pvs --noheading -o pv_name); do
        info "finding all vgs in $PV"
        for VG in $(lvm vgs --noheading -o vg_name -S "pv_name=$PV"); do
            echo "finding all lvs in $VG"
            for LV in $(lvm lvs --noheading -o lv_name -S "vg_name=$VG"); do
                decrypted_dev=$(dmsetup info -c | awk "/${LV}.*LUKS/ {print \$1}")
                [ -n "$decrypted_dev" ] && {
                    echo "$LV is encrypted and is open as $decrypted_dev. unmounting and closing it"
                    umount -l /dev/mapper/$decrypted_dev || true
                    cryptsetup luksClose $decrypted_dev || true
                }
                info "removing LV ${VG}/${LV}"
                umount "/dev/${VG}/${LV}" || true
                lvm lvchange -an "${VG}/${LV}" || true
                lvm lvremove -ff "${VG}/${LV}" || true
            done
            info "removing VG ${VG}"
            lvm vgchange -an "$VG" || true
            lvm vgremove -ff "$VG" || true
        done
        info "removing PV ${PV}"
        lvm pvremove -ff "$PV" || true
    done
    info "rescanning PVs to empty cache"
    lvm pvscan --cache || true

    usbinstalldev=$(mount | grep /run/install/repo | cut -f 1 -d ' ' | tr -d [:digit:] || true)
    DISKS=$(get-disks)
    info "Writing zeros to the start and end of all partitions on any non-removable media found in this list: $DISKS"
    for DEV in $DISKS ; do
        DEV=${DEV##*/}
        [ -d /sys/block/$DEV ] || continue

        # Skip removable disks (USB bootflash)
        REMOVABLE=$(</sys/block/$DEV/removable)
        if (( $REMOVABLE == 1 )); then
            info "Not zeroing removable device $DEV"
            continue
        fi
        if [ "$usbinstalldev" = "/dev/${DEV}" ]; then
            info "Not zeroing out USB install media $DEV"
            continue
        fi
        for DEVPART in $(ls /dev/${DEV}* | sort -r); do
            umount $DEVPART || :
            wipe_blockdev $DEVPART 4
        done
        udevadm settle
        blockdev --rereadpt "/dev/$DEV"
        udevadm settle
    done
    lvm pvscan --cache || true
}

assert_root
wipe_lvs