jberezanski/ValidateStorageHardware.ps1

## ValidateStorageHardware.ps1
######################################################################
######### STORAGE SPACES PHYSICAL DISK VALIDATION SCRIPT 2.0 #########
######################################################################
#
# [Description]:
# This script validates key functional requirements and performance
# characteristics of physical disks which are required to achieve a
# minimally functional storage deployment for general-purpose
# workloads. Use this script along with the Cluster Validation
# Wizard as the first steps of a thorough validation and profiling
# plan.
#
# [Cautionary Usage Notes]:
# While this script attempts to cover a comprehensive set of tests
# to validate storage hardware, it is not a replacement for
# thorough and planned evaluation of a new storage deployment.
# There are no guarantees expressed or implied resulting from the
# output of this script.
#
# This script is licensed for use "as-is" under Microsoft's Limited
# Public License version 1.1, which among other conditions, has the
# user bear all risks in its use. In the course of running this
# script, drives are initialized, partitioned, and formatted to
# prepare for performance testing - this will result in data loss.
# **Only use this script in a pre-production environment which has
# no risk of data loss for business-critical data.**
#
# [Usage]:
# For the most up to date usage notes check the Script Center:
# https://gallery.technet.microsoft.com/scriptcenter/
# Storage-Spaces-Physical-7ca9f304
#
# Modifications by Jakub Bereżański (j.berezanski@medicalgorithmics.com) based on version from 2015-07-26:
# - not relying on PhysicalDisk.DeviceID/Disk.Number; using SerialNumber for mapping from PhysicalDisk to Disk instead
# - added diagnostic messages to function Prepare-Disk
# - restoring original MPIO policy at end of script
# - corrected message wrongly claiming that MPIO policy was being set to Round Robin (the script sets Least Blocks)
# - disabled adjustment of MPIO policy on other cluster nodes
#
######################################################################
##### For questions, please contact spaces_deploy@microsoft.com ######
######################################################################

<#
.SYNOPSIS
Tests storage hardware for Spaces-readiness.

.DESCRIPTION
Validates the hardware configuration to determine if it is ready to deploy Storage Spaces for enterprise workloads. The following checks are performed:
- hardware and firmware checked against blacklists
- hardware and firmware checked for consistency
- disk drives checked for health status, canPool status, media and connection interface type
- Disables write-caching on HDDs
- MPIO load-balance policy settings are tested and set to maximize mixed IOPS
- Basic disk performance testing to identify drives which should be replaced

.PARAMETER OutputPath
Sends output to the specified file. If blank, output is sent to the consoloe.

.PARAMETER OutputCSV
If true, output will also be logged to comma-separated-value files. The filenames are derived from the OutputPath variable using the format <$OutputPath>_N.csv, where N is the next available integer. If no OutputPath is provided, but OutputCSV is selected, the default CSV filename will be <$OutputPath>_N.csv.

.EXAMPLE
PS>> .\ValidateStorageHardware.ps1
Basic default execution. Outputs all results to the console.

.EXAMPLE
PS>> .\ValidateStroageHardware.ps1 -OutputPath "C:\Users\User1\out.txt"
Sends output to the file out.txt. Console output is suppressed.

#>

Param (
#Sends output to the specified file. If blank, output is sent to the console.
[String]$OutputPath = "",
[bool]$OutputCSV = $false, #if true, data will also be written in CSV format to <$OutputPath>_N.csv, where N is the next available integer

# PARAMETERS TO ENSURE ARE CORRECT BEFORE RUNNING TEST

# By default will use all the disks in the system which can be pooled
[Object]$PhysicalDisks = (Get-PhysicalDisk |? {$_.CanPool -eq $True}),

# Need to specify a drive letter not in use as fsutil does not support mount points, ensure not in use
[char]$TempDriveLetter = 'T',

# The root mount folder, ensure not in use
[String]$MountPoint = "C:\TestVolumes",
[String]$MountPointPrefix = "Volume",
# The folder to write the results to. Ensure this is not in use.
[String]$ResultsTemp = ".\spaces_validation_temp",

################################
####### TEST PARAMETERS ########
################################
# Do change the thresholds as you see fit from your own modeling of the hardware in your environment
# SSD parameters vary dramatically depending on the type of interface, type of flash, and capacity
# HDD paramters vary depending on the rotation speed
# 7.2K - 3 QD/HDD
# 10K  - 4-5 QD/HDD
# 15K  - 6 QD/HDD
[int]$Duration = 60 * 2, # Number of seconds to perform each perf test

[int]$ThreadsHDD = 1, # Number of I/O threads per disk
[int]$QueueDepthHDD = 3, # Number of outstanding I/O per disk
[int]$ThreadsSSD = 2, # Number of I/O threads per disk
[int]$QueueDepthSSD = 10, # Number of outstanding I/O per disk

[int]$ThreadsHDDSeq = 1, # Number of I/O threads per disk
[int]$QueueDepthHDDSeq = 3, # Number of outstanding I/O per disk
[int]$ThreadsSSDSeq = 1, # Number of I/O threads per disk
[int]$QueueDepthSSDSeq = 8, # Number of outstanding I/O per disk)

[int]$MixedWritePercentage = 30, # Percentage of writes (reads are 100-this)

[bool]$TestSequential = $false, # Additional detailed testing for sequential I/O (optional)

##############################################
###### WARN/FAIL Performance Thresholds ######
##############################################
# Note: The current thresholds are gathered from a sample of Enterprise-class HDDs and SSDs available to the author
# Do change the thresholds as you see fit from your own modeling of the hardware in your environment
# Latency percentiles are far more accurate than Min, Avg, Max for demonstrating the performance of storage hardware
# Profile the hardware with the "-L" switch in Diskspd to get the detailed latency percentiles
# (99.999%-ile latency is labelled "5 nines")
[int]$SSD50thLatencyWarnThreshold = 5,
[int]$SSD50thLatencyFailThreshold = 10,

[int]$HDD50thLatencyWarnThreshold = 35,
[int]$HDD50thLatencyFailThreshold = 50,

[int]$SSD90thLatencyWarnThreshold = 10,
[int]$SSD90thLatencyFailThreshold = 20,

[int]$HDD90thLatencyWarnThreshold = 65,
[int]$HDD90thLatencyFailThreshold = 90,

[int]$SSD95thLatencyWarnThreshold = 35,
[int]$SSD95thLatencyFailThreshold = 65,

[int]$HDD95thLatencyWarnThreshold = 85,
[int]$HDD95thLatencyFailThreshold = 120,

[int]$WearWarnThreshold = 25,
[int]$WearFailThreshold = 75,

[double]$MPIOThreshold = 0.9, # Performance threshold to mark drives which have reduced Round Robin (RR) performance

# FOR DEVELOPMENT PURPOSES ONLY - DO NOT CHANGE
[bool]$VeryQuickPrecondition = $false, # Override the automatic precondition time with a quick time (30s)
[bool]$ForceUseExisting = $false, # Use existing disks in the $MountPoint\ directory, regardless of PhysicalDisks param
[bool]$SkipCleanup = $false, # Don't cleanup the disks, test volumes, mount points, and results folder
[bool]$UseBatching = $true, # Use the new batching method to speed up run times
[int]$MaxBatchSizeHDD = 20,
[int]$MaxBatchSizeSSD = 6,
[bool]$SkipMPIO = $false,
[bool]$SkipSetCache = $false
)

function Get-SD {
    param ([double[]]$Numbers)

    if ($Numbers.Count -lt 2) {
        $sd = 0
        $sd
    } else {
        $avg = $numbers | Measure-Object -Average | select Count, Average
        $popdev = 0

        foreach ($number in $numbers) {
            $popdev +=  [math]::pow(($number - $avg.Average), 2)
        }

        $sd = [math]::sqrt($popdev / ($avg.Count-1))
        $sd
    }
}

function Get-Avg {
    param ([double[]]$Numbers)

    $avg = $numbers | Measure-Object -Average | select Count, Average
    $avg.Average
}

function PrintBanner {
    param ([String]$BannerText, [int]$width = 80)

    # First Line of #'s
    $str = ("=" * $width + "`r`n")

    # 2nd line, centered banner text
    $padding = [int](($width - ($BannerText.Length + 2)) / 2)
    if ($BannerText.Length % 2 -eq 1) {
        $str += ("="*$padding + " " + $BannerText + " " + "="*($padding-1))
    } else {
        $str += ("="*$padding + " " + $BannerText + " " + "="*($padding))
    }

    # 3rd line of #'s
    $str += ("`r`n" + "=" * $width)

    PrintH $str
}

function PrintH {
    param (
        [Parameter(Mandatory=$true,ValueFromPipeline=$true)]
        [Object[]]$InputObject
    )
    begin {
        $objects = @()
    }

    process {
        $objects += $InputObject
    }

    end {
        if ($objects[0].getType() -eq [String]) {
            $str = $objects
        } elseif ($OutputPath -eq "") {
            $str = $objects | format-table -AutoSize
        } else {
            $str = $objects | format-table -AutoSize | out-string -Width 1024
        }

        if ($OutputPath -eq "") {
            $str
        } else {
            $str | Out-File -FilePath $OutputPath -Append
        }

        if ($OutputCSV -and $objects[0].GetType() -ne [String]) {
            $nextInt = ls "$($OutputPath)_*" | measure | select Count
            $objects | Select * | Export-Csv -Path "$($OutputPath)_$($nextInt.Count).csv"
        }
    }
}

function Prepare-Disk {
    param ([Object]$disk)

    PrintH (">> Preparing disk $($disk.Number) ($($disk.Manufacturer) $($disk.Model) $($disk.SerialNumber))")
    $mount_path = "$MountPoint\$MountPointPrefix" + $disk.UniqueId
    $temp_mount_path = "$TempDriveLetter`:"

    $err = New-Item -Path $mount_path -ItemType Directory -Force -Confirm:$false

    # Initialize the disk and format with a file system
    PrintH (">>> Setting IsReadOnly to false")
    $err = $disk | Set-Disk -IsReadOnly $false
    PrintH (">>> Setting IsOffline to false")
    $err = $disk | Set-Disk -IsOffline $false
    PrintH (">>> Initializing the disk")
    $err = $disk | Initialize-Disk -PartitionStyle GPT -ErrorAction Ignore
    PrintH (">>> Creating a partition")
    $partition = $disk | New-Partition -UseMaximumSize
    PrintH (">>> Formatting the partition")
    $volume = $partition | Format-Volume -FileSystem NTFS -Force -Confirm:$false

    # Give a moment for the volume to propegate
    Start-Sleep -Seconds 1

    PrintH (">>> Assigning temp drive letter")
    #FSUTIL does not support mount points, so need to assign a temporary drive letter to create test files
    $err = (Get-WmiObject Win32_Volume | Where {$_.Name -like $volume.Path}).AddMountPoint($ExecutionContext.InvokeCommand.ExpandString($temp_mount_path))

    $size = $volume.SizeRemaining - 4GB

    PrintH (">>> Creating test files")
    # create test files - this is a shortcut to having sqlio zero out the file with extending writes
    $err = fsutil file createnew $temp_mount_path\testfile $size
    $err = fsutil file setvaliddata $temp_mount_path\testfile $size

    PrintH (">>> Removing the temp drive letter")
    # remove the temporary drive letter
    $err = $partition | Remove-PartitionAccessPath -AccessPath $temp_mount_path

    PrintH (">>> Mounting the volume in the root mount folder")
    # mount the volume in the root mount folder
    $err = (Get-WmiObject Win32_Volume | Where {$_.Name -like $volume.Path}).AddMountPoint($ExecutionContext.InvokeCommand.ExpandString($mount_path))
}

function Test-All-Disks-Random {
    param ([Object[]]$disks, [int]$mix, [bool]$Async = $false)

    $file_path = ""

    $disks | ForEach-Object {
        $file_path += "$MountPoint\$MountPointPrefix" + $_.UniqueId + "\testfile "
    }


    if ($disks.MediaType -eq "HDD") {
        $QD = $QueueDepthHDD
        $Threads = $ThreadsHDD
        $MT = "HDD"
    } else {
        $QD = $QueueDepthSSD
        $Threads = $ThreadsSSD
        $MT = "SSD"
    }

    $output_path = "$ResultsTemp\$MT-random-w$mix.txt"

    if (!$Async) {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$duration -t$Threads -o$QD -b8k -w$mix -r8k -L -h $file_path" -Wait -WindowStyle Hidden -RedirectStandardOutput $output_path
        [int](Select-Number -path $output_path -pattern "total:" -index 3)
    } else {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$duration -t$Threads -o$QD -b8k -w$mix -r8k -L -h $file_path" -WindowStyle Hidden -RedirectStandardOutput $output_path
    }
}

function Test-All-Disks-Sequential {
    param ([Object[]]$disks, [int]$mix, [bool]$Async = $false)

    $file_path = ""

    $disks | ForEach-Object {
        $file_path += "$MountPoint\$MountPointPrefix" + $_.UniqueId + "\testfile "
    }

    if ($disks.MediaType -eq "HDD") {
        $QD = $QueueDepthHDDSeq
        $Threads = $ThreadsHDDSeq
        $MT = "HDD"
    } else {
        $QD = $QueueDepthSSDSeq
        $Threads = $ThreadsSSDSeq
        $MT = "SSD"
    }

    $output_path = "$ResultsTemp\$MT-sequential-w$mix.txt"


    if (!$Async) {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$duration -t$Threads -o$QD -b256k -w$mix -si -L -h $file_path" -Wait -WindowStyle Hidden -RedirectStandardOutput $output_path
        [int](Select-Number -path $output_path -pattern "total:" -index 2)
    } else {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$duration -t$Threads -o$QD -b256k -w$mix -si -L -h $file_path" -WindowStyle Hidden -RedirectStandardOutput $output_path
    }
}

function Test-Global-Disks-Random {
    param ([Object[]]$hdd, [Object[]]$ssd, [int]$mix)

    Test-All-Disks-Random -disks $hdd -mix $mix -Async $true
    Test-All-Disks-Random -disks $ssd -mix $mix -Async $true

    while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
        Start-Sleep -Seconds 5
    }

    $IOPS = 0

    $IOPS += [int](Select-Number -path ("$ResultsTemp\HDD-random-w$mix.txt") -pattern "total:" -index 3)
    $IOPS += [int](Select-Number -path ("$ResultsTemp\SSD-random-w$mix.txt") -pattern "total:" -index 3)

    $IOPS
}

function Test-Global-Disks-Sequential {
    param ([Object[]]$hdd, [Object[]]$ssd, [int]$mix)

    Test-All-Disks-Sequential -disks $hdd -mix $mix -Async $true
    Test-All-Disks-Sequential -disks $ssd -mix $mix -Async $true

    while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
        Start-Sleep -Seconds 5
    }

    $MBPS = 0

    $MBPS += [int](Select-Number -path ("$ResultsTemp\HDD-sequential-w$mix.txt") -pattern "total:" -index 2)
    $MBPS += [int](Select-Number -path ("$ResultsTemp\SSD-sequential-w$mix.txt") -pattern "total:" -index 2)

    $MBPS
}

function Measure-Disks-Performance-MPIO {
    param ([Object[]]$Disks, [String]$MPIOPolicy)

    if ($MPIOPolicy -ne "N/A" ) {Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $MPIOPolicy }

    # Measure sequential/random read/write performance in aggregate
    $Throughput = "" | Select Model, FirmwareVersion, NumberOfDisks, LoadBalancePolicy, SequentialWrite, SequentialRead, WriteIOPS, ReadIOPS, MixedIOPS, EVALUATION, EVALREASON

    if ($MPIOPolicy -eq "RR") {
        $Throughput.LoadBalancePolicy = "Round Robin"
    } elseif ($MPIOPolicy -eq "LB") {
        $Throughput.LoadBalancePolicy = "Least Blocks"
    } else {
        $Throughput.LoadBalancePolicy = "N/A"
    }

    $Throughput.Evaluation = "PASS"
    $Throughput.EvalReason = ""

    $Throughput.NumberOfDisks = ($Disks.Count)

    if ( (@($Disks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)})).Count -gt 1 ) {

        $Throughput.Model = "Multiple"
        $Throughput.FirmwareVersion = "Multiple"

        $HDD = $Disks |? {$_.MediaType -eq "HDD"}
        $SSD = $Disks |? {$_.MediaType -ne "HDD"}

        $Throughput.SequentialRead = Test-Global-Disks-Sequential -hdd $hdd -ssd $ssd -mix 0
        $Throughput.SequentialWrite = Test-Global-Disks-Sequential -hdd $hdd -ssd $ssd -mix 100

        $Throughput.ReadIOPS = Test-Global-Disks-Random -hdd $hdd -ssd $ssd -mix 0
        $Throughput.WriteIOPS = Test-Global-Disks-Random -hdd $hdd -ssd $ssd -mix 100
        $Throughput.MixedIOPS = Test-Global-Disks-Random -hdd $hdd -ssd $ssd -mix $MixedWritePercentage

    } else {
        $Throughput.Model = ($Disks[0]).Model
        $Throughput.FirmwareVersion = ($Disks[0]).FirmwareVersion

        $Throughput.SequentialRead = Test-All-Disks-Sequential -disks $Disks -mix 0
        $Throughput.SequentialWrite = Test-All-Disks-Sequential -disks $Disks -mix 100

        $Throughput.ReadIOPS = Test-All-Disks-Random -disks $Disks -mix 0
        $Throughput.WriteIOPS = Test-All-Disks-Random -disks $Disks -mix 100
        $Throughput.MixedIOPS = Test-All-Disks-Random -disks $Disks -mix $MixedWritePercentage
    }


    $Throughput
}

function Evaluate-MPIO-Performance {
    param ([Object[]]$Disks, [double]$Threshold = 0.9, [bool]$mpio = $false)

    if ($mpio) { # If there's MPIO on the system, do the full test comparing RR to LB
        $RR = Measure-Disks-Performance-MPIO -Disks $Disks -MPIOPolicy "RR"
        $LB = Measure-Disks-Performance-MPIO -Disks $Disks -MPIOPolicy "LB"

        if (($RR.SequentialWrite -lt $TSW * $Threshold) -or
            ($RR.SequentialRead -lt $TSR * $Threshold)) {
            $RR.Evaluation = "WARN"
            $RR.EvalReason += "There might be throughput limitations; "
        } elseif ((($RR.SequentialWrite -lt 2250) -and ($TSW -gt 2250)) -or
            (($RR.SequentialRead -lt 2250) -and ($TSR -gt 2250))) {
            $RR.Evaluation = "WARN"
            $RR.EvalReason += "Lower perf than single SAS cable; "
        }

        if (($RR.SequentialWrite -lt $LB.SequentialWrite * $Threshold) -or
            ($RR.SequentialRead -lt $LB.SequentialRead * $Threshold) -or
            ($RR.WriteIOPS -lt $LB.WriteIOPS * $Threshold) -or
            ($RR.ReadIOPS -lt $LB.ReadIOPS * $Threshold)) {
            $RR.Evaluation = "FAIL"
            $RR.EvalReason += "Reduced RR perf; "
        }

        @($RR, $LB)
    } else { # Otherwise we're just looking to test aggregate performance
        $Batch = Measure-Disks-Performance-MPIO -Disks $Disks -MPIOPolicy "N/A"

        @($Batch)
    }
}

function Precondition-Disk {
    param ([Object]$disk, [int]$SingleQDThroughput, [bool]$async = $true)

    $file_path = "$MountPoint\$MountPointPrefix" + $disk.UniqueId + "\"
    $partition = (Get-Partition |? {($_.AccessPaths -ne $null) -and ($_.AccessPaths).Contains($file_path)})
    $file_path += "testfile"

    if ($partition -eq $null) {
        PrintH (">> There was a problem with preconditioning drive " + $disk.UniqueId)
        PrintH "Clean the configuration and restart the script."
        exit
    }

    $output_path = "$ResultsTemp\"+$disk.UniqueId+"-precondition.txt"

    if (!$VeryQuickPrecondition) {
        [long]$time = 2 * [long](($partition.Size / 1MB) / $SingleQDThroughput)
    } else {
        [long]$time = 30
    }


    # Now precondition the drive
    if ($async) {
        PrintH (">> Preconditioning in parallel " + $disk.Model + " (" + $disk.SerialNumber + ") for " + $time + " seconds...")
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList ("-d$time -t1 -o1 -b256k -w100 -si -L -h $file_path") -WindowStyle Hidden -RedirectStandardOutput $output_path
    } else {
        PrintH (">> Preconditioning " + $disk.Model + " (" + $disk.SerialNumber + ") for " + $time + " seconds...")
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList ("-d$time -t1 -o1 -b256k -w100 -si -L -h $file_path") -WindowStyle Hidden -Wait -RedirectStandardOutput $output_path
    }
}

function Test-Disk-Sequential {
    param ([Object]$disk, [int]$mix, [bool]$precondition = $false, [bool]$Async = $false)

    $file_path = "$MountPoint\$MountPointPrefix" + $disk.UniqueId + "\testfile"
    $output_path = "$ResultsTemp\"+$disk.UniqueId+"-seq-w$mix.txt"

    New-Item -Path $output_path -ItemType File -Force | Out-Null

    if (!$precondition) {
        if ($disk.MediaType -eq "HDD") {
            $qd = $QueueDepthHDDSeq
            $threads = $ThreadsHDDSeq
        } else {
            $qd = $QueueDepthSSDSeq
            $threads = $ThreadsSSDSeq
        }
    } else { # precondition settings
        $qd = 1
        $threads = 1
    }

    if ($Async) {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$Duration -t$threads -o$qd -b256k -w$mix -si -L -n -h $file_path" -RedirectStandardOutput $output_path -WindowStyle Hidden
    } else {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$Duration -t$threads -o$qd -b256k -w$mix -si -L -n -h $file_path" -Wait -RedirectStandardOutput $output_path -WindowStyle Hidden
    }

    if (!$Async) {
        if (($precondition) -and ($mix -eq 100)) {
            Parse-Disk-Sequential-Precondition -disk $disk
        } else {
            Parse-Disk-Sequential -disk $disk -mix $mix
        }
    }
}

function Parse-Disk-Sequential {
    param ([Object]$disk, [int]$mix)

    $output_path = "$ResultsTemp\"+$disk.UniqueId+"-seq-w$mix.txt"

    if ($mix -eq 100) {
        $disk.SequentialWrite       = Select-Number -path $output_path -pattern "total:" -index 2
    } else {
        $disk.SequentialRead        = Select-Number -path $output_path -pattern "total:" -index 2
    }
}

function Parse-Disk-Sequential-Precondition {
    param ([Object]$disk)

    $output_path = "$ResultsTemp\"+$disk.UniqueId+"-seq-w$mix.txt"

    $disk.SequentialWritePrecondition       = Select-Number -path $output_path -pattern "total:" -index 2
}

function Test-Disk-Random {
    param ([Object]$disk, [int]$mix, [bool]$Async = $false)

    $file_path = "$MountPoint\$MountPointPrefix" + $disk.UniqueId + "\testfile"
    $output_path = "$ResultsTemp\"+$disk.UniqueId+"-w$mix.txt"
    New-Item -Path $output_path -ItemType File -Force | Out-Null

    if ($disk.MediaType -eq "HDD") {
        $qd = $QueueDepthHDD
        $threads = $ThreadsHDD
    } else {
        $qd = $QueueDepthSSD
        $threads = $ThreadsSSD
    }

    if ($Async) {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$Duration -t$threads -o$qd -b8k -w$mix -r8k -L -n -h $file_path" -RedirectStandardOutput $output_path -WindowStyle Hidden
    } else {
        Start-Process -FilePath ".\diskspd.exe" -ArgumentList "-d$Duration -t$threads -o$qd -b8k -w$mix -r8k -L -n -h $file_path" -Wait -RedirectStandardOutput $output_path -WindowStyle Hidden
    }

    if (!$Async) { # If we're running synchronously, we can immediately process the output, otherwise the caller will need to parse after the batch is done
        Parse-Disk-Random -disk $disk -mix $mix
    }
}

function Parse-Disk-Random {
    param ([Object]$disk, [int]$mix)

    $output_path = "$ResultsTemp\"+$disk.UniqueId+"-w$mix.txt"

    if ($mix -eq 100) {
        $disk.WriteIOPs       = Select-Number -path $output_path -pattern "total:"
        $disk.WriteLatency50 = Select-Number -path $output_path -pattern "50th"
        $disk.WriteLatency90 = Select-Number -path $output_path -pattern "90th"
        $disk.WriteLatency95 = Select-Number -path $output_path -pattern "95th"
    } elseif ($mix -eq 0) {
        $disk.ReadIOPs       = Select-Number -path $output_path -pattern "total:"
        $disk.ReadLatency50 = Select-Number -path $output_path -pattern "50th"
        $disk.ReadLatency90 = Select-Number -path $output_path -pattern "90th"
        $disk.ReadLatency95 = Select-Number -path $output_path -pattern "95th"
    } else {
        $disk.MixedIOPs       = Select-Number -path $output_path -pattern "total:"
        $disk.MixedLatency50 = Select-Number -path $output_path -pattern "50th"
        $disk.MixedLatency90 = Select-Number -path $output_path -pattern "90th"
        $disk.MixedLatency95 = Select-Number -path $output_path -pattern "95th"
    }
}

function Select-Number {
    param ([String]$path, [String]$pattern, [int]$index = 3) # index = 3 happens to works for both IOPS and latency, index = 2 works for throughput

    if (!(Test-Path -Path $path)) {
        $null
        return
    }

    $number = Select-String -Path $path -Pattern $pattern -SimpleMatch

    # record data for IOP CSV
    if ($number -ne $null) {
        $ret = [double]($number[0].Line.Split("|")[$index].Trim())
        $ret
    } else {
        -1
    }
}

function FindOutliers {
    param (
        [double[]]$numbers,
        [double]$sd_threshold = 1.0, # report values outside this many sd's from the mean
        [double]$sd_min = 0.03, # minimum the sd can be as a percent of the average, don't bother going through the
        [double]$sd_max = 0.4, # maximum the sd can be as a percent of the average, report all drives if this condition occurs
        [double]$noise = 0.015, # maximum amount of noise around average is tolerated
        [bool]$above) # if true, then report outliers above the mean, false otherwise

    $sd = [double](Get-SD -Numbers $numbers)
    $avg = [double](Get-Avg -Numbers $numbers)

    [int[]]$indexes = @()

    # Check if none or all disks need to be reported
    if ($sd -lt ($sd_min * $avg)) { # too low sd
        return @()
    } elseif ($sd -gt ($sd_max * $avg)) { # too high sd
        if (($sd - $avg) -gt 5) { # filter out when the numbers are low
            return (0..($numbers.Count-1))
        }
    }

    # Check individual disks
    for ($i = 0; $i -lt $numbers.Count; $i++) {
        if ($above) {
            if (($numbers[$i] -gt ($avg + ($noise * $avg))) -and ($numbers[$i] -gt ($avg + ($sd * $sd_threshold)))) {
                $indexes += @($i)
            }
        } else {
            if (($numbers[$i] -lt ($avg - ($noise * $avg))) -and ($numbers[$i] -lt ($avg - ($sd * $sd_threshold)))) {
                $indexes += @($i)
            }
        }
    }

    $indexes
}

function MarkModelsWithHighVariance {
    param (
        [Object[]]$Disks,
        [double]$Threshold)

    $Disks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
        $group = $_.Group
        $warned = $false
        $numbers = ($group | Select ReadIOPS).ReadIOPS

        $sd = [double](Get-SD -Numbers $numbers)
        $avg = [double](Get-Avg -Numbers $numbers)

        if (($avg * $Threshold) -lt $sd) {
            $group | ForEach-Object {
                WarnDisk -Disk $_ -Reason "Drive model has high read variance"
                $warned = $true
            }
        }

        $numbers = ($group | Select WriteIOPS).WriteIOPS

        $sd = [double](Get-SD -Numbers $numbers)
        $avg = [double](Get-Avg -Numbers $numbers)

        if (($avg * $Threshold) -lt $sd) {
            $group | ForEach-Object {
                WarnDisk -Disk $_ -Reason "Drive model has high write variance"
                $warned = $true
            }
        }

        $numbers = ($group | Select MixedIOPS).MixedIOPS

        $sd = [double](Get-SD -Numbers $numbers)
        $avg = [double](Get-Avg -Numbers $numbers)

        if (($avg * $Threshold) -lt $sd) {
            $group | ForEach-Object {
                WarnDisk -Disk $_ -Reason "Drive model has high mixed variance"
                $warned = $true
            }
        }
    }
}

function CheckMPIOPaths {
    param ([Object[]]$Disks)

    $MPIODisks = devcon find "MPIO\DISK*"
    $SCSIDisks = devcon find "SCSI\DISK*"

    [String[]]$DiskModels = @()

    $Disks | ForEach-Object {
        if (!($DiskModels.Contains($_.Model))) {
            $DiskModels += @($_.Model)
        }
    }

    $DisksInfo = @()

    $DiskModels | ForEach-Object {
        $scsi = $SCSIDisks | Select-String -Pattern ($_.Trim()) -SimpleMatch
        $mpio = $MPIODisks | Select-String -Pattern ($_.Trim()) -SimpleMatch

        $temp = "" | Select Model, TotalDisks, TotalPaths, AvgPathsPerDisk, EVALUATION, EvalReason
        $temp.Model = $_
        $temp.TotalPaths = ($scsi.Count - $mpio.Count)
        $temp.TotalDisks = $mpio.Count
        $temp.AvgPathsPerDisk = $temp.TotalPaths / $temp.TotalDisks

        # This does not save to the main TestDisks variable to store results, so just hold it temporarily
        if ($temp.AvgPathsPerDisk -eq 1) {
            $temp.EVALUATION = "PASS"
        } else {
            $temp.EVALUATION = "WARN"
            $temp.EvalReason = "Only a single path detected. Add more paths for redundancy and performance."
        }

        $DisksInfo += @($temp)
    }

    $DisksInfo
}

function PassDisk {
    param ([Object]$Disk, [String]$Reason)

    if (($Disk.EVALUATION -eq "PASS") -and ($Reason -ne $null)) {
        $Disk.EVALUATION = "PASS*"
    }

    if ($Reason -ne $null) {
        $Disk.EvalReason += "$Reason; "
    }
}

function WarnDisk {
    param ([Object]$Disk, [String]$Reason)

    if ($Disk.EVALUATION -ne "FAIL") {
        $Disk.EVALUATION = "Warn"
    }

    $Disk.EvalReason += "$Reason; "
}

function FailDisk {
    param ([Object]$Disk, [String]$Reason)

    $Disk.EVALUATION = "FAIL"
    $Disk.EvalReason += "$Reason; "
}

function ConvertMPIOLoadPolicy {
    param ([String]$Policy)

    $PolicyMap = @{
        "Round Robin" = "RR"
        "Least Blocks" = "LB"
    }
    if ($Policy -in $PolicyMap.Keys) {
        $PolicyMap[$Policy]
    } else {
        $Policy
    }
}

########################################################################
############################## MAIN ####################################
########################################################################

if ((Get-Command ".\diskspd.exe") -eq $null) {
    PrintH "ERROR: diskspd.exe not found. Put diskspd.exe in the same directory as this script."
    exit
}

if ((Get-Command ".\dskcache.exe") -eq $null) {
    PrintH "ERROR: dskcache.exe not found. \
    Download it from https://support.microsoft.com/kb/811392 \
    Put it in the same directory as this script."
    exit
}

Stop-Process -Name "diskspd" -Force -ErrorAction Ignore
$PDCache = Get-PhysicalDisk
$DCache = Get-Disk

if ($ForceUseExisting) {
    $PhysicalDisks = @()

    Get-Item -Path $MountPoint | Get-ChildItem -Filter "$MountPointPrefix*" | ForEach-Object {
        $num = [int]([regex]::Match($_.Name, "$MountPointPrefix(\d+)")).Groups[1].Value

        $pd = $PDCache |? {$_.DeviceId -eq $num}

        if ($pd -ne $null) {
            $PhysicalDisks += @($pd)
        }

        $pd
    }
}

if ($PhysicalDisks.Count -eq 0) {
    PrintH "ERROR: There are no disks to test."
    exit
}

$TestDisks = @()

$PhysicalDisks | ForEach-Object {
    $disk = $_ | Select FriendlyName, DeviceId, UniqueId,
        BusType, CanPool, CannotPoolReason,
        EnclosureNumber, SlotNumber, PhysicalLocation,
        Manufacturer, Model, SerialNumber, FirmwareVersion,
        HealthStatus, OperationalStatus, IsPartial,
        MediaType,
        LogicalSectorSize, PhysicalSectorSize, Size,
        ReadIOPS, ReadLatency50, ReadLatency90, ReadLatency95,
        WriteIOPS, WriteLatency50, WriteLatency90, WriteLatency95,
        MixedIOPS, MixedLatency50, MixedLatency90, MixedLatency95,
        ReadLatencyMax_c, WriteLatencyMax_c, FlushLatencyMax_c,
        Wear, ReadErrorsTotal, WriteErrorsTotal,
        PD,
        SequentialRead, SequentialWrite, SequentialWritePrecondition,
        Evaluation, EvalReason

    $disk.PD = $_

    if ($disk.Model -ne $null) { $disk.Model = ($disk.Model).Trim() }
    if ($disk.SerialNumber -ne $null) { $disk.SerialNumber = ($disk.SerialNumber).Trim() }
    if ($disk.FirmwareVersion -ne $null) { $disk.FirmwareVersion = ($disk.FirmwareVersion).Trim() }

    $disk.Evaluation = "PASS"
    $disk.EvalReason = ""

    $TestDisks += @($disk)
}

######################################################################
PrintBanner -BannerText "VALIDATING HARDWARE"

[int]$DiskModelWarningThreshold = 4
$uDiskModels = $PhysicalDisks | Select Model -Unique
if ($uDiskModels.Count -ge $DiskModelWarningThreshold) {
    PrintH (">> Warning: $DiskModelWarningThreshold or more unique disk models detected")
} else {
    PrintH ">> Disk drives satisfy hardware recommendations."
}

######################################################################
PrintBanner -BannerText "VALIDATING FIRMWARE"

$diskFwBlacklist = @()
$badFW = $PhysicalDisks | Where {$diskFwBlacklist.Contains($_.FirmwareVersion)}
if ($badFW.Count -gt 0) {
    PrintH ">> Error: The following disks have blacklisted firmware"
    $badFW | Select FriendlyName, Model, SerialNumber, FirmwareVersion, BusType | PrintH
    exit
}

$fwVersions = @{}
$badModels = @()
$PhysicalDisks | ForEach-Object {
    if (-not $fwVersions.Contains($_.Model)) {
        $fwVersions.Add($_.Model, $_.FirmwareVersion)
    } elseif (-not $fwVersions[$_.Model].Equals($_.FirmwareVersion)) {
        $badModels += $_.Model
    }
}
$diffFWDisks =  $PhysicalDisks | Where {$badModels.contains($_.Model)}
if ($diffFWDisks.Count -gt 0) {
    PrintH ">> Error: The following disks have different firmware"
    $diffFWDisks | Select FriendlyName, Model, SerialNumber, FirmwareVersion, BusType | PrintH
    exit
}

PrintH ">> All given disks have identical, non-blacklisted firmware."

if (Get-Command Get-StorageEnclosure -ErrorAction SilentlyContinue) {
    $jbodFwBlacklist = @()
    $badFW = Get-StorageEnclosure | Where {$jbodFwBlacklist.Contains($_.FirmwareVersion)}
    if ($badFW.Count -gt 0) {
        PrintH ">> Error: The following storage enclosures have blacklisted firmware"
        $badFW | Select FriendlyName, Model, SerialNumber, FirmwareVersion | PrintH
        exit
    }

    $fwVersions = @{}
    $badModels = @()
    Get-StorageEnclosure | ForEach-Object {
        if (-not $fwVersions.Contains($_.Model)) {
            $fwVersions.Add($_.Model, $_.FirmwareVersion)
        } elseif (-not $fwVersions[$_.Model].Equals($_.FirmwareVersion)) {
            $badModels += $_.Model
        }
    }
    $diffFWJbods =  Get-StorageEnclosure | Where {$badModels.contains($_.Model)}
    if ($diffFWJbods.Count -gt 0) {
        PrintH ">> Error: The following storage enclosures have different firmware"
        $diffFWJbods | Select FriendlyName, Model, SerialNumber, FirmwareVersion | PrintH
        exit
    }
    PrintH ">> All given storage enclosures have identical, non-blacklisted firmware."
} else {
    PrintH ">> Get-StorageEnclosure not installed on this computer. Skipping StorageEnclosure firmware validation."
}

PrintH ">> Firmware validation checks passed."

######################################################################
PrintBanner -BannerText "VALIDATING HEALTH"

$NotHealthy = $TestDisks |? {$_.HealthStatus -ne "Healthy"}

if ($NotHealthy.Count -gt 0) {
    PrintH ">> Error: The following disks are not healthy."
    $NotHealthy | Select FriendlyName, Model, SerialNumber, FirmwareVersion, HealthStatus | PrintH
    exit
}

PrintH ">> All given disks are healthy."

######################################################################
PrintBanner -BannerText "VALIDATING BUS TYPE"

$NonSASDisks = $TestDisks |? {($_.BusType -ne "SAS") -and ($_.BusType -ne "SATA") -and ($_.BusType -ne "NVMe")}

if ($NonSASDisks.Count -gt 0) {
    PrintH ">> Error: The following disks are not SAS, SATA, or NVMe"
    $NonSASDisks | Select FriendlyName, Model, SerialNumber, FirmwareVersion, BusType | PrintH
} else {
    PrintH ">> All given disks are SAS, SATA, or NVME."
}

######################################################################
if (!$ForceUseExisting) {
    PrintBanner -BannerText "VALIDATING DISKS ARE POOLABLE AND NOT PARTIAL"

    $CannotPool = $TestDisks |? {!($_.CanPool)}

    if ($CannotPool.Count -gt 0) {
        PrintH ">> Error: The following disks cannot be pooled:"
        $CannotPool | Select FriendlyName, Model, SerialNumber, FirmwareVersion, CanPool, CannotPoolReason | PrintH
        exit
    }

    $IsPartial = $TestDisks |? {$_.IsPartial}

    if ($IsPartial.Count -gt 0) {
        PrintH ">> Error: The following disks are partial:"
        $IsPartial | Select FriendlyName, Model, SerialNumber, FirmwareVersion, IsPartial | PrintH
        exit
    }

    PrintH ">> All given disks can be pooled and are not partial."
}

######################################################################
PrintBanner -BannerText "ADJUSTING WRITE CACHE SETTINGS"

if (!$SkipSetCache) {
	$hddDiskNums = $PhysicalDisks | % { $this = $_; $DCache | ? {($_.SerialNumber).Contains($this.SerialNumber) } } | select Number
    foreach ($num in $hddDiskNums) {
        Start-Process -FilePath ".\dskcache.exe" -ArgumentList "-w PhysicalDrive$($num.Number)" -WindowStyle Hidden
    }

    $unspecifiedMedia = $PhysicalDisks | where {$_.MediaType -eq "UnSpecified"}
    if ($unspecifiedMedia.Count -gt 0) {
        PrintH ">> Warning: The following disks can't be identified as SSD or HDD"
        $unspecifiedMedia | Select FriendlyName, Model, SerialNumber, FirmwareVersion, MediaType | PrintH
    }

    PrintH ">> Write cache settings confirmed"
} else {
    PrintG ">> Write cache settings skipped"
}

######################################################################
PrintBanner -BannerText "CHECKING MPIO SETTINGS"

$mpio = $true
$LoadPolicy = "LB"
if ((Get-WindowsFeature -Name Multipath-IO).InstallState -ne "Installed") {
    PrintH ">> Warning: MPIO Feature is not Installed on this computer. Multipath-specific tests will not be run."
    $mpio = $false
} elseif ($SkipMPIO) {
    PrintH ">> User requested MPIO tests to be skipped."
    $mpio = $false
} else {
    if (!(Get-MSDSMAutomaticClaimSettings).Item("SAS")) {
        PrintH ">> Warning: MPIO Feature is installed, but SAS devices are not claimed. Multipath-specific tests will not be run."
        $mpio = $false
    } else {
        $OriginalLoadPolicy = Get-MSDSMGlobalDefaultLoadBalancePolicy
        PrintH ">> MPIO Feature is installed and SAS devices are claimed. Setting global load balance policy to ${LoadPolicy} (from $OriginalLoadPolicy)."

        Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy
    }
}

if ($mpio -and ((Get-Command devcon -ErrorAction SilentlyContinue) -ne $null)) {
    PrintH ">> The following are all the disk models attached to this system, and number of paths:"
    $DiskPathInfo = CheckMPIOPaths -Disks $PhysicalDisks

    $DiskPathInfo | PrintH
    $NonPathRedundantDisks = @($Disks) |? {$_.AvgPathsPerDisk -lt 2}

    if ($NonPathRedundantDisks.Count -gt 0) {
        PrintH "Warning: The following disks models do not have redundant paths:"

        $NonPathRedundantDisks | ForEach-Object {
            WarnDisk -Disk $_ -Reason "No redundant I/O paths"
        }

        $NonPathRedundantDisks | PrintH
    }
}

######################################################################
PrintBanner -BannerText "PREPARING DISKS FOR PERFORMANCE TESTING"

if (!(Test-Path -Path $ResultsTemp)) {
    $err = New-Item -Path $ResultsTemp -ItemType Directory -Confirm:$false -Force

    if ($err -eq $null) {
        PrintH ">> Could not create the directory for the test results."
        PrintH ">> If the directory exists, try deleting it, or choose a new directory name."
        exit
    }
}

if ($mpio) { Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy }

PrintH ">> Preparing disks - this will take some time."

# Prepare disks with a volume
$TestDisks | ForEach-Object {
    $this = $_

    if (($this.Model -eq "Virtual Disk") -or ($this.SerialNumber -eq $null) -or (($this.SerialNumber).Trim() -eq "")) { # No serial number, or a VHD, use the unreliable method to associate PD to Disk
        $disk = $this.PD | Get-Disk
    } else { # Otherwise will need to get the association based on serial numbers (since they're very likely unique)
        $disk = $DCache |? {($_.SerialNumber).Contains($this.SerialNumber)}
    }

    if ($disk -ne $null) {
        if (!$ForceUseExisting) {
            Prepare-Disk $disk
        }
    } else {
        PrintH (">> Error while preparing" + $_.FriendlyName + "-" + $_.Manufacturer + $_.Model + "(" + $_.SerialNumber + ")")
        PrintH ">> This likely implies a hardware issue. Try running the script without the drive."
        exit
    }
}

######################################################################
PrintBanner -BannerText "MEASURE SINGLE DRIVE PERFORMANCE FROM EACH MODEL"

if ($mpio) { Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy }

$SingleThroughputArray = @()

$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $disk = $_.Group[0]
    Test-Disk-Sequential -disk $disk -mix 100 -precondition $true

    if ($UseBatching) {
        Test-Disk-Sequential -disk $disk -mix 0
        Test-Disk-Sequential -disk $disk -mix 100
        Test-Disk-Random -Disk $disk -Mix 0
        Test-Disk-Random -Disk $disk -Mix 100
        Test-Disk-Random -Disk $disk -Mix $MixedWritePercentage
    }

    $SingleThroughputArray += @($disk)
}

if ($mpio) { Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy }

$SingleThroughputArray | Sort-Object {$_.Model, $_.FirmwareVersion, $_.SerialNumber} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, SequentialRead, SequentialWrite, ReadIOPS, WriteIOPS, MixedIOPS | PrintH

######################################################################
PrintBanner -BannerText "PRECONDITONING SSD DRIVES FOR RANDOM I/O"

if ($mpio) { Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy }

if (($TestDisks |? {$_.MediaType -ne "HDD"}).Count -gt 0) {

    # Then precondition the drive for the necessary time to overwrite the size twice
    $TestDisks | ForEach-Object {
        if ($_.MediaType -ne "HDD") {
            $disk = $_
            $single_throughput = -1

            $SingleThroughputArray | ForEach-Object {
                if (($single_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.SequentialWrite -gt 0)) {
                    $single_throughput = $_.SequentialWrite
                }
            }

            if ($single_throughput -gt 0) {
                Precondition-Disk -disk $disk -SingleQDThroughput $single_throughput -async $true
            } else {
                PrintH (">> There was an issue attempting to precondition " + $disk.Model + " with serial number " + $disk.SerialNumber)
                PrintH ">> This likely implies a hardware issue. Try running the script without the drive."
                exit
            }
        }
    }

    # Wait for the drives to finish
    while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
        Start-Sleep -Seconds 5
    }
} else {
    PrintH ">> No SSDs to Precondition."
}

######################################################################
PrintBanner -BannerText "MEASURING OVERALL SYSTEM THROUGHPUT"

if ($mpio) { Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy }

if (@($TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)}).Count -gt 1) {
    $Throughput_ALL = Evaluate-MPIO-Performance -Disks $TestDisks -Threshold $MPIOThreshold -mpio $mpio
    PrintH ">> Aggregate performance of all drives"

    if ($mpio) {
        $Throughput_ALL | Select Model, FirmwareVersion, NumberOfDisks, LoadBalancePolicy, SequentialRead, SequentialWrite, ReadIOPS, WriteIOPS, MixedIOPS | PrintH
    } else {
        $Throughput_ALL | Select Model, FirmwareVersion, NumberOfDisks, SequentialRead, SequentialWrite, ReadIOPS, WriteIOPS, MixedIOPS | PrintH
    }

    $ThroughputArray = @($Throughput_ALL)

    # Set default policy to LB because that is now the default
    if ($mpio) {
        $BestThroughput = $ThroughputArray | Sort-Object -Property MixedIOPS -Descending | select -First 1
        #$LoadPolicy = ConvertMPIOLoadPolicy -Policy $BestThroughput.LoadBalancePolicy
        $LoadPolicy = "LB"
    }

} else {
    $ThroughputArray = @()
}

$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Throughput_Group = Evaluate-MPIO-Performance -Disks $_.Group -Threshold $MPIOThreshold -mpio $mpio

    PrintH (">> Aggregate performance of " + $_.Name)
    if ($mpio) {
        $Throughput_Group | Select Model, FirmwareVersion, NumberOfDisks, LoadBalancePolicy, SequentialRead, SequentialWrite, ReadIOPS, WriteIOPS, MixedIOPS | PrintH
    } else {
        $Throughput_Group | Select Model, FirmwareVersion, NumberOfDisks, SequentialRead, SequentialWrite, ReadIOPS, WriteIOPS, MixedIOPS | PrintH
    }

    $ThroughputArray += @($Throughput_Group)

    if ($mpio) {
        $BestThroughput = $ThroughputArray | Sort-Object -Property MixedIOPS -Descending | select -First 1
        #$LoadPolicy = ConvertMPIOLoadPolicy -Policy $BestThroughput.LoadBalancePolicy
        $LoadPolicy = "LB"
    }
}

######################################################################
PrintBanner -BannerText "MEASURING INDIVIDUAL DRIVE PERFORMANCE"

if ($mpio) { Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy }

if ($UseBatching) { # THIS IS THE NEW BATCHED METHOD OF TESTING WHICH SIGNIFIGANTLY IMPROVES RUN TIMES

    $TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
        $group = $_.Group
        $disk = $group[0]
        $num_disks = $_.Count

        if ($TestSequential) {

            ###################################
            #### MEASURING SEQUENTIAL READ ####
            ###################################

            $aggr_throughput = -1
            $single_throughput = -1

            $ThroughputArray | ForEach-Object {
                if (($aggr_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.SequentialRead -gt 0)) {
                    $aggr_throughput = $_.SequentialRead
                }
            }

            $SingleThroughputArray | ForEach-Object {
                if (($single_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.SequentialRead -gt 0)) {
                    $single_throughput = $_.SequentialRead
                }
            }

            if (($single_throughput -eq -1) -or ($arrr_throughput -eq -1)) {
                PrintH ">> There was a problem running the workload test. Clear the configuration and re-run the script."
                exit
            }

            [int]$batch_size = 1

            # If there's enough throughput to run all the drives simultaniously
            if (($single_throughput * $num_disks) -lt ($aggr_throughput * 0.9)) {
                $batch_size = $num_disks
            } else { # Otherwise batch the run
                $batch_size = ($aggr_throughput * 0.9) / $single_throughput
            }

            if ($batch_size -lt 1) {
                $batch_size = 1
            } elseif (($disk.MediaType -eq "HDD") -and ($batch_size -gt $MaxBatchSizeHDD)) {
                $batch_size = $MaxBatchSizeHDD
            } elseif (($disk.MediaType -ne "HDD") -and ($batch_size -gt $MaxBatchSizeSSD)) {
                $batch_size = $MaxBatchSizeSSD
            }

            for ($i = 0; $i -lt $num_disks; $i += $batch_size) {
                $end = $i + $batch_size - 1

                if ($end -gt $num_disks) {
                    $end = $num_disks - 1
                }

                $group[$i..$end] | ForEach-Object { Test-Disk-Sequential -disk $_ -mix 0 -Async $true }
                while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
                    Start-Sleep -Seconds 5
                }
                $group[$i..$end] | ForEach-Object { Parse-Disk-Sequential -disk $_ -mix 0 }
            }

            ####################################
            #### MEASURING SEQUENTIAL WRITE ####
            ####################################

            $aggr_throughput = -1
            $single_throughput = -1

            $ThroughputArray | ForEach-Object {
                if (($aggr_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.SequentialWrite -gt 0)) {
                    $aggr_throughput = $_.SequentialWrite
                }
            }

            $SingleThroughputArray | ForEach-Object {
                if (($single_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.SequentialWrite -gt 0)) {
                    $single_throughput = $_.SequentialWrite
                }
            }

            if (($single_throughput -eq -1) -or ($arrr_throughput -eq -1)) {
                PrintH ">> There was a problem running the workload test. Clear the configuration and re-run the script."
                exit
            }

            [int]$batch_size = 1

            # If there's enough throughput to run all the drives simultaniously
            if (($single_throughput * $num_disks) -lt ($aggr_throughput * 0.9)) {
                $batch_size = $num_disks
            } else { # Otherwise batch the run
                $batch_size = ($aggr_throughput * 0.9) / $single_throughput
            }

            if ($batch_size -lt 1) {
                $batch_size = 1
            } elseif (($disk.MediaType -eq "HDD") -and ($batch_size -gt $MaxBatchSizeHDD)) {
                $batch_size = $MaxBatchSizeHDD
            } elseif (($disk.MediaType -ne "HDD") -and ($batch_size -gt $MaxBatchSizeSSD)) {
                $batch_size = $MaxBatchSizeSSD
            }

            for ($i = 0; $i -lt $num_disks; $i += $batch_size) {
                $end = $i + $batch_size - 1

                if ($end -gt $num_disks) {
                    $end = $num_disks - 1
                }

                $group[$i..$end] | ForEach-Object { Test-Disk-Sequential -disk $_ -mix 100 -Async $true }
                while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
                    Start-Sleep -Seconds 5
                }
                $group[$i..$end] | ForEach-Object { Parse-Disk-Sequential -disk $_ -mix 100 }
            }
        }

        ###############################
        #### MEASURING RANDOM READ ####
        ###############################

        $aggr_throughput = -1
        $single_throughput = -1

        $ThroughputArray | ForEach-Object {
            if (($aggr_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.ReadIOPS -gt 0)) {
                $aggr_throughput = $_.ReadIOPS
            }
        }

        $SingleThroughputArray | ForEach-Object {
            if (($single_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.ReadIOPS -gt 0)) {
                $single_throughput = $_.ReadIOPS
            }
        }

        if (($single_throughput -eq -1) -or ($arrr_throughput -eq -1)) {
            PrintH ">> There was a problem running the workload test. Clear the configuration and re-run the script."
            exit
        }

        [int]$batch_size = 1

        # If there's enough throughput to run all the drives simultaniously
        if (($single_throughput * $num_disks) -lt ($aggr_throughput * 0.9)) {
            $batch_size = $num_disks
        } else { # Otherwise batch the run
            $batch_size = ($aggr_throughput * 0.9) / $single_throughput
        }

        if ($batch_size -lt 1) {
            $batch_size = 1
        } elseif (($disk.MediaType -eq "HDD") -and ($batch_size -gt $MaxBatchSizeHDD)) {
            $batch_size = $MaxBatchSizeHDD
        } elseif (($disk.MediaType -ne "HDD") -and ($batch_size -gt $MaxBatchSizeSSD)) {
            $batch_size = $MaxBatchSizeSSD
        }

        for ($i = 0; $i -lt $num_disks; $i += $batch_size) {
            $end = $i + $batch_size - 1

            if ($end -gt $num_disks) {
                $end = $num_disks - 1
            }
            $group[$i..$end] | ForEach-Object { Test-Disk-Random -disk $_ -mix 0 -Async $true }
            while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
                Start-Sleep -Seconds 5
            }
            $group[$i..$end] | ForEach-Object { Parse-Disk-Random -disk $_ -mix 0 }
        }

        ################################
        #### MEASURING RANDOM WRITE ####
        ################################

        $aggr_throughput = -1
        $single_throughput = -1

        $ThroughputArray | ForEach-Object {
            if (($aggr_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.WriteIOPS -gt 0)) {
                $aggr_throughput = $_.WriteIOPS
            }
        }

        $SingleThroughputArray | ForEach-Object {
            if (($single_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.WriteIOPS -gt 0)) {
                $single_throughput = $_.WriteIOPS
            }
        }

        if (($single_throughput -eq -1) -or ($arrr_throughput -eq -1)) {
            PrintH ">> There was a problem running the workload test. Clear the configuration and re-run the script."
            exit
        }

        [int]$batch_size = 1

        # If there's enough throughput to run all the drives simultaniously
        if (($single_throughput * $num_disks) -lt ($aggr_throughput * 0.9)) {
            $batch_size = $num_disks
        } else { # Otherwise batch the run
            $batch_size = ($aggr_throughput * 0.9) / $single_throughput
        }

        if ($batch_size -lt 1) {
            $batch_size = 1
        } elseif (($disk.MediaType -eq "HDD") -and ($batch_size -gt $MaxBatchSizeHDD)) {
            $batch_size = $MaxBatchSizeHDD
        } elseif (($disk.MediaType -ne "HDD") -and ($batch_size -gt $MaxBatchSizeSSD)) {
            $batch_size = $MaxBatchSizeSSD
        }

        for ($i = 0; $i -lt $num_disks; $i += $batch_size) {
            $end = $i + $batch_size - 1

            if ($end -gt $num_disks) {
                $end = $num_disks - 1
            }

            $group[$i..$end] | ForEach-Object { Test-Disk-Random -disk $_ -mix 100 -Async $true }
            while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
                Start-Sleep -Seconds 5
            }
            $group[$i..$end] | ForEach-Object { Parse-Disk-Random -disk $_ -mix 100 }
        }

        ################################
        #### MEASURING RANDOM MIXED ####
        ################################

        $aggr_throughput = -1
        $single_throughput = -1

        $ThroughputArray | ForEach-Object {
            if (($aggr_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.MixedIOPS -gt 0)) {
                $aggr_throughput = $_.MixedIOPS
            }
        }

        $SingleThroughputArray | ForEach-Object {
            if (($single_throughput -eq -1) -and ($disk.Model -eq $_.Model) -and ($disk.FirmwareVersion -eq $_.FirmwareVersion) -and ($_.MixedIOPS -gt 0)) {
                $single_throughput = $_.MixedIOPS
            }
        }

        if (($single_throughput -eq -1) -or ($arrr_throughput -eq -1)) {
            PrintH ">> There was a problem running the workload test. Clear the configuration and re-run the script."
            exit
        }

        [int]$batch_size = 1

        # If there's enough throughput to run all the drives simultaniously
        if (($single_throughput * $num_disks) -lt ($aggr_throughput * 0.9)) {
            $batch_size = $num_disks
        } else { # Otherwise batch the run
            $batch_size = ($aggr_throughput * 0.9) / $single_throughput
        }

        if ($batch_size -lt 1) {
            $batch_size = 1
        } elseif (($disk.MediaType -eq "HDD") -and ($batch_size -gt $MaxBatchSizeHDD)) {
            $batch_size = $MaxBatchSizeHDD
        } elseif (($disk.MediaType -ne "HDD") -and ($batch_size -gt $MaxBatchSizeSSD)) {
            $batch_size = $MaxBatchSizeSSD
        }

        for ($i = 0; $i -lt $num_disks; $i += $batch_size) {
            $end = $i + $batch_size - 1

            if ($end -gt $num_disks) {
                $end = $num_disks - 1
            }

            $group[$i..$end] | ForEach-Object { Test-Disk-Random -disk $_ -mix $MixedWritePercentage -Async $true }
            while ((Get-Process -Name "diskspd" -ErrorAction Ignore) -ne $null) {
                Start-Sleep -Seconds 5
            }
            $group[$i..$end] | ForEach-Object { Parse-Disk-Random -disk $_ -mix $MixedWritePercentage }
        }
    }
} else { # OLD SYNC METHOD OF TESTING, ONE AT A TIME - CAN POTENTIALLY DELIVER MORE STABLE NUMBERS
    # Measure random/sequential read/write performance for each drive
    $TestDisks | ForEach-Object {
        $_.EVALUATION = "PASS"
        Test-Disk-Sequential -disk $_ -mix 0
        Test-Disk-Sequential -disk $_ -mix 100
        Test-Disk-Random -Disk $_ -Mix 0
        Test-Disk-Random -Disk $_ -Mix 100
        Test-Disk-Random -Disk $_ -Mix $MixedWritePercentage
    }
}

# Check if latencies are high on HDDs and SSDs
$TestDisks | ForEach-Object {

    ###################################################
    ########### CHECKING 50%-ile LATENCIES ############
    ###################################################
    if ($_.MediaType -eq "HDD") {

        if (($_.ReadLatency50 -gt $HDD50thLatencyFailThreshold) -or ($_.WriteLatency50 -gt $HDD50thLatencyFailThreshold) -or ($_.MixedLatency50 -gt $HDD50thLatencyFailThreshold)) {
            FailDisk -Disk $_ -Reason "Very high 50%-ile latency"
        } elseif (($_.ReadLatency50 -gt $HDD50thLatencyWarnThreshold) -or ($_.WriteLatency50 -gt $HDD50thLatencyWarnThreshold) -or ($_.MixedLatency50 -gt $HDD50thLatencyWarnThreshold)) {
            WarnDisk -Disk $_ -Reason "High 50%-ile latency"
        }

    } else {

        if (($_.ReadLatency50 -gt $SSD50thLatencyFailThreshold) -or ($_.WriteLatency50 -gt $SSD50thLatencyFailThreshold) -or ($_.MixedLatency50 -gt $SSD50thLatencyFailThreshold)) {
            FailDisk -Disk $_ -Reason "Very high 50%-ile latency"
        } elseif (($_.ReadLatency50 -gt $SSD50thLatencyWarnThreshold) -or ($_.WriteLatency50 -gt $SSD50thLatencyWarnThreshold) -or ($_.MixedLatency50 -gt $SSD50thLatencyWarnThreshold)) {
            WarnDisk -Disk $_ -Reason "High 50%-ile latency"
        }

    }

    ###################################################
    ########## CHECKING 90%-ile LATENCIES #############
    ###################################################
    if ($_.MediaType -eq "HDD") {

        if (($_.ReadLatency90 -gt $HDD90thLatencyFailThreshold) -or ($_.WriteLatency90 -gt $HDD90thLatencyFailThreshold) -or ($_.MixedLatency90 -gt $HDD90thLatencyFailThreshold)) {
            FailDisk -Disk $_ -Reason "Very high 90%-ile latency"
        } elseif (($_.ReadLatency90 -gt $HDD90thLatencyWarnThreshold) -or ($_.WriteLatency90 -gt $HDD90thLatencyWarnThreshold) -or ($_.MixedLatency90 -gt $HDD90thLatencyWarnThreshold)) {
            WarnDisk -Disk $_ -Reason "High 90%-ile latency"
        }

    } else {

        if (($_.ReadLatency90 -gt $SSD90thLatencyFailThreshold) -or ($_.WriteLatency90 -gt $SSD90thLatencyFailThreshold) -or ($_.MixedLatency90 -gt $SSD90thLatencyFailThreshold)) {
            FailDisk -Disk $_ -Reason "Very high 90%-ile latency"
        } elseif (($_.ReadLatency90 -gt $SSD90thLatencyWarnThreshold) -or ($_.WriteLatency90 -gt $SSD90thLatencyWarnThreshold) -or ($_.MixedLatency90 -gt $SSD90thLatencyWarnThreshold)) {
            WarnDisk -Disk $_ -Reason "High 90%-ile latency"
        }

    }

    #######################################################
    ########### CHECKING 95%-ile LATENCIES ############
    #######################################################
    if ($_.MediaType -eq "HDD") {

        if (($_.ReadLatency95 -gt $HDD95thLatencyFailThreshold) -or ($_.WriteLatency95 -gt $HDD95thLatencyFailThreshold) -or ($_.MixedLatency95 -gt $HDD95thLatencyFailThreshold)) {
            FailDisk -Disk $_ -Reason "Very high 95%-ile latency"
        } elseif (($_.ReadLatency95 -gt $HDD95thLatencyWarnThreshold) -or ($_.WriteLatency95 -gt $HDD95thLatencyWarnThreshold) -or ($_.MixedLatency95 -gt $HDD95thLatencyWarnThreshold)) {
            WarnDisk -Disk $_ -Reason "High 95%-ile latency"
        }

    } else {

        if (($_.ReadLatency95 -gt $SSD95thLatencyFailThreshold) -or ($_.WriteLatency95 -gt $SSD95thLatencyFailThreshold) -or ($_.MixedLatency95 -gt $SSD95thLatencyFailThreshold)) {
            FailDisk -Disk $_ -Reason "Very high 95%-ile latency"
        } elseif (($_.ReadLatency95 -gt $SSD95thLatencyWarnThreshold) -or ($_.WriteLatency95 -gt $SSD95thLatencyWarnThreshold) -or ($_.MixedLatency95 -gt $SSD95thLatencyWarnThreshold)) {
            WarnDisk -Disk $_ -Reason "High 95%-ile latency"
        }

    }
}

if ($TestSequential) {
    # Check consistancy of sequential read
    $TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
        $Group = $_.Group

        FindOutliers -numbers (($Group | Select SequentialRead).SequentialRead) -above $false | ForEach-Object {
            FailDisk -Disk $Group[[int]$_] -Reason "Outlier for sequential read"
        }
    }

    # Check consistancy of sequential write
    $TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
        $Group = $_.Group

        FindOutliers -numbers (($Group | Select SequentialWrite).SequentialWrite) -above $false | ForEach-Object {
            FailDisk -Disk $Group[[int]$_] -Reason "Outlier for sequential write"
        }
    }

    $TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion, $_.ReadIOPS} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, SequentialRead, SequentialWrite | PrintH
}

# Check consistancy of read IOPS
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select ReadIOPS).ReadIOPS) -above $false | ForEach-Object {
        FailDisk -Disk $Group[[int]$_] -Reason "Outlier for read IOPS"
    }
}

$TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion, $_.ReadIOPS} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, ReadIOPs, ReadLatency50, ReadLatency90, ReadLatency95 | PrintH

# Check consistancy of write IOPS
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select WriteIOPS).WriteIOPS) -above $false | ForEach-Object {
        FailDisk -Disk $Group[[int]$_] -Reason "Outlier for write IOPS"
    }
}

$TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion, $_.WriteIOPS} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, WriteIOPs, WriteLatency50, WriteLatency90, WriteLatency95 | PrintH

# Check consistancy of mixed IOPS
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select MixedIOPS).MixedIOPS) -above $false | ForEach-Object {
        FailDisk -Disk $Group[[int]$_] -Reason "Outlier for mixed IOPS"
    }
}

$TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion, $_.MixedIOPS} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, MixedIOPS, MixedLatency50, MixedLatency90, MixedLatency95 | PrintH

# Mark drive models with high variance
MarkModelsWithHighVariance -Disks $TestDisks -Threshold 0.5

######################################################################
PrintBanner -BannerText "CHECKING RELIABILITY COUNTERS (LATENCY)"

$TestDisks | ForEach-Object {
    $rel = $_.PD | Get-StorageReliabilityCounter

    $_.ReadLatencyMax_c    =   $rel.ReadLatencyMax
    $_.WriteLatencyMax_c   =   $rel.WriteLatencyMax
    $_.FlushLatencyMax_c   =   $rel.FlushLatencyMax
    $_.Wear                =   $rel.Wear
    $_.ReadErrorsTotal     =   $rel.ReadErrorsTotal
    $_.WriteErrorsTotal    =   $rel.WriteErrorsTotal
}

# Check consistancy of read latency max
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select ReadLatencyMax_c).ReadLatencyMax_c) -above $true -sd_threshold 2.5 | ForEach-Object {
        WarnDisk -Disk $Group[[int]$_] -Reason "Outlier for read latency max"
    }
}

# Check consistancy of write latency max
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select WriteLatencyMax_c).WriteLatencyMax_c) -above $true -sd_threshold 2.5 | ForEach-Object {
        WarnDisk -Disk $Group[[int]$_] -Reason "Outlier for write latency max"
    }
}

# Check consistancy of flush latency max
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select FlushLatencyMax_c).FlushLatencyMax_c) -above $true -sd_threshold 2.5 | ForEach-Object {
        WarnDisk -Disk $Group[[int]$_] -Reason "Outlier for flush latency max"
    }
}

$TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, ReadLatencyMax_c, WriteLatencyMax_c, FlushLatencyMax_c | PrintH

######################################################################
PrintBanner -BannerText "CHECKING RELIABILITY COUNTERS (HEALTH)"

# Check consistancy of read errors
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select ReadErrorsTotal).ReadErrorsTotal) -above $true | ForEach-Object {
        WarnDisk -Disk $Group[[int]$_] -Reason "Outlier for read errors"
    }
}

# Check consistancy of write errors
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select WriteErrorsTotal).WriteErrorsTotal) -above $true | ForEach-Object {
        WarnDisk -Disk $Group[[int]$_] -Reason "Outlier for write errors"
    }
}

# Check wear consistancy
$TestDisks | Group-Object {("" + $_.Model + ";" + $_.FirmwareVersion)} | ForEach-Object {
    $Group = $_.Group

    FindOutliers -numbers (($Group | Select Wear).Wear) -above $true | ForEach-Object {
        FailDisk -Disk $Group[[int]$_] -Reason "Outlier for wear"
    }
}

# Check absolute wear
$TestDisks | ForEach-Object {
    $disk = $_

    if ($disk.Wear -ne $null) {
        if ($disk.Wear -ge $WearFailThreshold) {
            FailDisk -Disk $disk -Reason "Very high wear"
        } elseif ($disk.Wear -gt $WearWarnThreshold) {
            WarnDisk -Disk $disk -Reason "High wear"
        }
    }
}

$TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, Wear, ReadErrorsTotal, WriteErrorsTotal | PrintH

######################################################################
PrintBanner -BannerText "SUMMARY REPORT, SORTED BY EVALUATION"

$TestDisks | Sort-Object {$_.EVALUATION, $_.EvalReason, $_.Model, $_.FirmwareVersion} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, EVALUATION, EvalReason | PrintH

######################################################################
PrintBanner -BannerText "SUMMARY REPORT, SORTED BY MODEL, FIRMWARE"

$TestDisks | Sort-Object {$_.Model, $_.FirmwareVersion} | Select FriendlyName, Model, SerialNumber, FirmwareVersion, EVALUATION, EvalReason | PrintH

######################################################################
PrintBanner -BannerText "PERFORMING CLEANUP"

if ($mpio) {
    PrintH ">> Restoring MPIO global load balance policy to $OriginalLoadPolicy"
    Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $OriginalLoadPolicy
#    Get-ClusterNode | ForEach-Object {
#        Invoke-Command -ComputerName $_.Name -ScriptBlock {
#            Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy
#        }
#    }
}

#if ($dskCache) {
#    Get-ClusterNode | ForEach-Object {
#        Invoke-Command -ComputerName $_.Name -ScriptBlock {
#            Set-MSDSMGlobalDefaultLoadBalancePolicy -Policy $LoadPolicy
#        }
#    }
#}

if ($ForceUseExisting) {
    $SkipCleanup = $true
}

if (!$SkipCleanup) {
    $PhysicalDisks | % { $this = $_; $DCache | ? {($_.SerialNumber).Contains($this.SerialNumber) } } | Clear-Disk -RemoveData -Confirm:$false
    $err = [System.IO.Directory]::Delete($MountPoint, $true)
    Remove-Item -Path "$ResultsTemp" -Force -Recurse | Out-Null
    PrintH ">> Done."
} else {
    PrintH ">> Skipping."
}

PrintH ">> Exiting."