Performance test for pwsh -Login

pwshPerfTest.ps1

param(
    [int]$Iterations = 100,
    [switch]$Clean
)

$ErrorActionPreference = 'Stop'

function Exec
{
    param(
        [scriptblock]$sb,
        [switch]$IgnoreExitcode,
        [switch]$VerboseOutputOnError
    )
    $backupEAP = $script:ErrorActionPreference
    $script:ErrorActionPreference = "Continue"
    try {
        if($VerboseOutputOnError.IsPresent)
        {
            $output = & $sb 2>&1
        }
        else
        {
            & $sb
        }

        # note, if $sb doesn't have a native invocation, $LASTEXITCODE will
        # point to the obsolete value
        if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
            if($VerboseOutputOnError.IsPresent -and $output)
            {
                $output | Out-String | Write-Verbose -Verbose
            }

            # Get caller location for easier debugging
            $caller = Get-PSCallStack -ErrorAction SilentlyContinue
            if($caller)
            {
                $callerLocationParts = $caller[1].Location -split ":\s*line\s*"
                $callerFile = $callerLocationParts[0]
                $callerLine = $callerLocationParts[1]

                $errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
                throw $errorMessage
            }
            throw "Execution of {$sb} failed with exit code $LASTEXITCODE"
        }
    } finally {
        $script:ErrorActionPreference = $backupEAP
    }
}

function Get-TimeResults
{
    param([string]$Path)

    $results = [System.Collections.Generic.List[double]]::new()
    foreach ($entry in Get-Content -Path $Path)
    {
        $results.Add($entry)
    }

    return $results
}

$dotnetRuntime = if ($IsWindows -ne $false)
{ 
    throw "Can only run performance tests on Linux or macOS"
}
elseif ($IsLinux)
{
    'linux-x64'
}
else
{
    'osx-x64'
}

$pwshExePathInRepo = "src/powershell-unix/bin/Release/netcoreapp3.0/$dotnetRuntime/pwsh"

# Clone PowerShell master
$psBaseRoot = "$PSScriptRoot/PowerShell"
$pwshBaseExePath = "$psBaseRoot/$pwshExePathInRepo"
if ($Clean)
{
    Remove-Item -Force -Recurse $psBaseRoot -ErrorAction Ignore
}
if (-not (Test-Path $psBaseRoot))
{
    try
    {
        Exec { git clone "https://github.com/PowerShell/PowerShell" --branch master --single-branch $psBaseRoot }
        Push-Location $psBaseRoot
        Import-Module -Force "$psBaseRoot/build.psm1"
        Start-PSBootstrap
        Start-PSBuild -Clean -Configuration Release -CrossGen
    }
    finally
    {
        Pop-Location
    }
}

# Clone login prototype
$psloginRoot = "$PSScriptRoot/PowerShell-login"
$pwshLoginExePath = "$psloginRoot/$pwshExePathInRepo"
if ($Clean)
{
    Remove-Item -Force -Recurse $psloginRoot -ErrorAction Ignore
}
if (-not (Test-Path $psloginRoot))
{
    try
    {
        Exec { git clone "https://github.com/rjmholt/PowerShell" --branch exec-login --single-branch $psloginRoot }
        Push-Location $psloginRoot
        Import-Module -Force "$psloginRoot/build.psm1"
        Start-PSBuild -Clean -Configuration Release -CrossGen
        $pwshLoginExePath = Get-PSOutput
    }
    finally
    {
        Pop-Location
    }
}

# Create /bin/sh login script
$pwshLoginShPath = "$PSScriptRoot/pwsh-login.sh"
if ($Clean)
{
    Remove-Item -Force $pwshLoginShPath -ErrorAction Ignore
}
if (-not (Test-Path $pwshLoginShPath))
{
    Set-Content -Force -Path $pwshLoginShPath -Value @"
#!/bin/sh -l
exec $pwshBaseExePath "`$@"
"@
}
chmod +x $pwshLoginShPath

# Create a default C# consoleapp for reference
$csharpBaselineName = 'csharp-baseline'
$csharpBaselineRoot = "$PSScriptRoot/$csharpBaselineName"
$csharpBaselineExePath = "$csharpBaselineRoot/bin/Release/netcoreapp3.0/$dotnetRuntime/$csharpBaselineName"
if ($Clean)
{
    Remove-Item -Force $csharpBaselineRoot -ErrorAction Ignore
}
if (-not (Test-Path $csharpBaselineName))
{
    $csharpBaselineCsprojPath = "$csharpBaselineRoot/$csharpBaselineName.csproj"
    mkdir $csharpBaselineRoot
    Push-Location $csharpBaselineRoot
    try
    {
        Exec { dotnet new console }
        $csproj = Get-Content -Raw $csharpBaselineCsprojPath | ForEach-Object { [System.Xml.Linq.XDocument]::Parse($_) }
        $r2rElement = [System.Xml.Linq.XElement]::new("PublishReadyToRun", "true")
        $csproj.Element('Project').Element('PropertyGroup').Add($r2rElement)
        Set-Content -Path $csharpBaselineCsprojPath -Value $csproj
        Exec { dotnet publish -c Release -r $dotnetRuntime }
    }
    finally
    {
        Pop-Location
    }
}

# Set up performance data files
$csharpBaselineOutputPath = "$PSScriptRoot/CSharpBaseline.csv"
$pwshBaseOutputPath = "$PSScriptRoot/PwshBase.csv"
$pwshLoginBaseOutputPath = "$PSScriptRoot/PwshLoginBase.csv"
$pwshLoginShOutputPath = "$PSScriptRoot/PwshLoginSh.csv"
$pwshLoginLoginOutputPath = "$PSScriptRoot/PwshLoginLogin.csv"

foreach ($outputFile in $csharpBaselineOutputPath,$pwshBaseOutputPath,$pwshLoginBaseOutputPath,$pwshLoginShOutputPath,$pwshLoginLoginOutputPath)
{
    Remove-Item -Path $outputFile -Force -ErrorAction Ignore
}

# Template the bash performance runner script
$bashRunnerScript = @"
function get_real_time
{
    { TIMEFORMAT='%R' ; time `$1 "`${@:2}" ; } 2>&1 >/dev/null
}

function run_perf_loop
{
    bin_path=`$1
    for i in $(seq 1 $Iterations); do
        get_real_time `$bin_path "`${@:2}"
    done
}

PWSH_ARGS=(-NoProfile -NoLogo -Command '"0"')
PWSH_LOGIN_ARGS=(-Login "`${PWSH_ARGS[@]}")

run_perf_loop "$csharpBaselineExePath" > $csharpBaselineOutputPath
run_perf_loop "$pwshBaseExePath" "`${PWSH_ARGS[@]}" > "$pwshBaseOutputPath"
run_perf_loop "$pwshLoginExePath" "`${PWSH_ARGS[@]}" > "$pwshLoginBaseOutputPath"
run_perf_loop "$pwshLoginShPath" "`${PWSH_ARGS[@]}" > "$pwshLoginShOutputPath"
run_perf_loop "$pwshLoginExePath" "`${PWSH_LOGIN_ARGS[@]}" > "$pwshLoginLoginOutputPath"
"@

# Run the script
Exec { /bin/bash -c $bashRunnerScript }

# Collate the results
$resultTable = @{
    'C# Baseline' = Get-TimeResults -Path $csharpBaselineOutputPath | Measure-Object -Average -StandardDeviation
    'pwsh base' = Get-TimeResults -Path $pwshBaseOutputPath | Measure-Object -Average -StandardDeviation
    'pwsh base [login branch]' = Get-TimeResults -Path $pwshLoginBaseOutputPath | Measure-Object -Average -StandardDeviation
    'pwsh-login.sh' = Get-TimeResults -Path $pwshLoginShOutputPath | Measure-Object -Average -StandardDeviation
    'pwsh -Login [login branch]' = Get-TimeResults -Path $pwshLoginLoginOutputPath | Measure-Object -Average -StandardDeviation
}

$pwshBaseResult = $resultTable['pwsh base'].Average
$testResults = @()
foreach ($test in $resultTable.Keys)
{
    $value = $resultTable[$test]
    $testResults += [pscustomobject]@{
        Name = $test
        Count = $value.Count
        Average = $value.Average
        StdDev = $value.StandardDeviation
        '% of pwsh' = ($value.Average / $pwshBaseResult)*100
    }
}

$testResults | Format-Table