bondarenkod/Invoke-NuGetPackageCaching.ps1

## readme.md

      
    Raw
  

              readme.md
            
          
    Possible Solution for Azure Functions .NET Worker Issues

Issues Encountered

Failed to restore /home/vsts/work/1/s/CarbonCopy.PI.Functions/obj/Release/net8.0/WorkerExtensions/WorkerExtensions.csproj
This error is encountered during the build process of an Azure Functions project targeting .NET 8.0.
Related GitHub Issues


Azure/azure-functions-dotnet-worker#1864
Azure/azure-functions-dotnet-worker#1888

The workaround


performs nugets caching before the function is being build
can cache nugets from private vsts feed using the already included authorization
I've tried to use nuget+nugetauthprovider directly, didn't work on ubuntu, probably due to hardcoded mono msbuild version in the nuget (or auth) tool.

20-JUL-2024 - haven't tested it fully yet, just finished. But the main build process is reported as OK.


## Invoke-NuGetPackageCaching.ps1

function Add-NugetAuthConfig {
    param (
        [Parameter(Mandatory = $true)]
        [string]$path,

        [Parameter(Mandatory = $true)]
        [string]$key,

        [Parameter(Mandatory = $true)]
        [string]$login,

        [Parameter(Mandatory = $true)]
        [string]$pwd
    )

    # Load the XML file
    [xml]$nugetConfig = Get-Content $path

    # Check if the packageSourceCredentials node exists, if not create it
    if ($null -eq $nugetConfig.configuration.packageSourceCredentials) {
        $packageSourceCredentialsNode = $nugetConfig.CreateElement("packageSourceCredentials")
        $nugetConfig.configuration.AppendChild($packageSourceCredentialsNode) | Out-Null
    }
    else {
        $packageSourceCredentialsNode = $nugetConfig.configuration.packageSourceCredentials
    }

    # Create the new package source credentials node
    $newPackageSourceNode = $nugetConfig.CreateElement($key)

    $usernameNode = $nugetConfig.CreateElement("add")
    $usernameNode.SetAttribute("key", "Username")
    $usernameNode.SetAttribute("value", $login)
    $newPackageSourceNode.AppendChild($usernameNode) | Out-Null

    $passwordNode = $nugetConfig.CreateElement("add")
    $passwordNode.SetAttribute("key", "ClearTextPassword")
    $passwordNode.SetAttribute("value", $pwd)
    $newPackageSourceNode.AppendChild($passwordNode) | Out-Null

    # Append the new package source credentials node
    $packageSourceCredentialsNode.AppendChild($newPackageSourceNode) | Out-Null

    # Save the updated XML back to the file
    $nugetConfig.Save($path)
    Write-Output "Updated nuget.config file saved to $path"
}

function Get-PasswordFromJson {
    param (
        [Parameter(Mandatory = $true)]
        [string]$JsonString,

        [Parameter(Mandatory = $true)]
        [string]$EndpointUrl
    )

    # Convert the JSON string to a PowerShell object
    $jsonObject = $JsonString | ConvertFrom-Json

    # Iterate through the endpointCredentials array to find the matching endpoint
    foreach ($credential in $jsonObject.endpointCredentials) {
        if ($credential.endpoint -eq $EndpointUrl) {
            return $credential.password
        }
    }

    # If no matching endpoint is found, return $null
    return $null
}

function Invoke-PackageEnumeration {
    param (
        [string]$csProjLookupDirectory
    )

    $csProjFiles = Get-ChildItem -Path $csProjLookupDirectory -Recurse -Filter *.csproj

    # Initialize the dictionary
    $packagesDictionary = @{}

    foreach ($file in $csProjFiles) {
        [xml]$xml = Get-Content -Path $file.FullName

        $packages = $xml.Project.ItemGroup.PackageReference

        foreach ($package in $packages) {
            $packageId = $package.Include
            $packageVersion = $package.Version

            if ([string]::IsNullOrEmpty($packageId)) {
                continue;
            }

            if ($false -eq $packagesDictionary.ContainsKey($packageId) ) {
                $packagesDictionary[$packageId] = @($packageVersion)
            }
            else {
                if (-not $packagesDictionary[$packageId].Contains($packageVersion)) {
                    $packagesDictionary[$packageId] += $packageVersion
                }
            }
        }
    }

    # Return the dictionary
    return $packagesDictionary
}

function Invoke-NuGetPackageCaching {
    param (
        [string]$config, # path to the nuget.config file, rel or abs
        [string[]]$feedKeys, # keys of the feeds to cache packages, should exist in the nuget.config file
        [string]$projectsDir = $null, # where to look for the csproj files, by default the directory of the nuget.config file
        [string]$nugetExePath = "nuget.exe" # path to the nuget.exe
    )


    $nugetConfigPath = Resolve-Path $config

    # check if projectsDir null or empty
    if ([string]::IsNullOrEmpty($projectsDir)) {
        $projectsDir = Split-Path $nugetConfigPath
    }

    if (-Not (Test-Path -Path $nugetExePath)) {
        Write-Host "Trying to resolve the nuget.exe path: '$nugetExePath'"
        $nugetPath = (Get-Command nuget.exe).Source
        Write-Host "Resolved nuget.exe path: '$nugetPath'"
        $nugetExePath = $nugetPath
    }

    Write-Warning "Prerequisites:"
    Write-Warning "nuget.exe should be installed on the machine (NuGetToolInstaller task)."
    Write-Warning "NuGetAuthenticate task should be executed before this task."

    # write the input parameters
    Write-Host "The Function Input parameters:"
    Write-Host "  config: $config"
    Write-Host "  feedKeys: $($feedKeys -join ', ')"
    Write-Host "  nugetExePath: $nugetExePath"
    Write-Host "  projectsDir: $projectsDir"

    # Get the NuGet global packages cache directory
    $globalPackagesCache = & dotnet nuget locals global-packages --list

    # Extract the path from the output
    $cachePath = $globalPackagesCache | Select-String -Pattern "global-packages: (.+)" | ForEach-Object { $_.Matches[0].Groups[1].Value }


    Write-Host "NuGet global packages list: $globalPackagesCache"
    Write-Host "NuGet global packages cache directory (parsed): $cachePath"

    $authProviderEnpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS
    Write-Host "NUGET_CREDENTIALPROVIDERS_PATH: $env:NUGET_CREDENTIALPROVIDERS_PATH"
    Write-Host "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS: $authProviderEnpoints"

    try {
        # VSS_NUGET_EXTERNAL_FEED_ENDPOINTS

        $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS = ""

        # Ensure the packages folder exists
        if (-Not (Test-Path -Path $cachePath)) {
            Write-Host "Creating the NuGet global packages cache directory at '$cachePath'"
            New-Item -ItemType Directory -Path $cachePath
        }

        # ensure the nuget.config file exists, exit with error if not
        if (-Not (Test-Path -Path $nugetConfigPath)) {
            Write-Error "NuGet config file not found at the provided path '$nugetConfigPath'"
            return 1
        }

        # parse the nuget.config XML file to get all the feeds
        $cfgFeedsDic = @{}
        $xml = [xml](Get-Content $nugetConfigPath)
        $xml.configuration.packageSources.add | ForEach-Object {
            $cfgFeedsDic[$_.key] = $_.value
        }

        # add auth info to the nuget.config file
        foreach ($key in $cfgFeedsDic.Keys) {
            $authPwd = Get-PasswordFromJson -JsonString $authProviderEnpoints -EndpointUrl $cfgFeedsDic[$key]
            if ([string]::IsNullOrEmpty($authPwd)) {
                Write-Host "The password for the feed '$key' is not found in the provided credentials. Skipping the feed..."
                continue;
            }
            Add-NugetAuthConfig -path $nugetConfigPath -key $key -login "vsts" -pwd $authPwd
        }

        # print the nuget.config file
        Write-Host "NuGet config file:"
        Get-Content $nugetConfigPath | ForEach-Object { Write-Host $_ }

        $cfgFeeds = @()
        # print the feed URLs and keys
        Write-Host "Exctracted feeds from the provided config:"
        foreach ($key in $cfgFeedsDic.Keys) {
            Write-Host "  $($key): $($cfgFeedsDic[$key])"
            $cfgFeeds += $cfgFeedsDic[$key]
        }

        $includedPackages = Invoke-PackageEnumeration -csProjLookupDirectory $projectsDir

        Write-Host "Used packages:"
        foreach ($ipKey in $includedPackages.Keys) {
            Write-Host "Package: $($ipKey):"
            foreach ($ipv in $includedPackages[$ipKey]) {
                Write-Host "  - $($ipv):"
            }
        }

        foreach ($feedKey in $feedKeys) {
            $feedUrlToCache = $cfgFeedsDic[$feedKey]
            Write-Host "Getting all the packages list from the feed: '$feedKey' - '$feedUrlToCache"
            # List all packages from the feed
            if ($IsWindows -eq $true) {
                $packages = & $nugetExePath list -Source $feedUrlToCache -ConfigFile $nugetConfigPath -NonInteractive -verbosity detailed
            }
            else {
                $packages = mono $nugetExePath list -Source $feedUrlToCache -ConfigFile $nugetConfigPath -NonInteractive -verbosity detailed
            }

            Write-Host "Caching packages from the feed: '$feedKey"

            foreach ($package in $packages) {
                Write-Host "Processing package: '$package'"
                if ($package -match "^(\S+)\s(\S+)$") {
                    $packageName = $matches[1]
                    $packageVersions = @($matches[2])

                    # if we dont have this package in the used one, we skip it

                    if ($false -eq $includedPackages.ContainsKey($packageName)) {
                        Write-Host "!!! The package '$packageName' not found in the included packages! Skipping it from caching..."
                        continue;
                    }

                    $includedVersions = $includedPackages[$packageName]

                    if ($includedVersions.count -gt 0) {
                        $packageVersions = $includedVersions
                    }

                    # caching all the listed package versions
                    foreach ($packageVersion in $packageVersions) {
                        Write-Host "Downloading package '$packageName.$packageVersion'"

                        if ($IsWindows -eq $true) {
                            & $nugetExePath install $packageName -Version $packageVersion -OutputDirectory $cachePath -ConfigFile $nugetConfigPath -NonInteractive -MSBuildPath $msbuildPath -verbosity detailed #-Source $cfgFeeds
                        }
                        else {
                            mono $nugetExePath install $packageName -Version $packageVersion -OutputDirectory $cachePath -ConfigFile $nugetConfigPath -NonInteractive -MSBuildPath $msbuildPath -verbosity detailed #-Source $cfgFeeds
                        }
                    }
                }
            }
        }
    }
    catch {
        Write-Error $_.Exception.Message
        return 1
    }
    finally {
        $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS = $authProviderEnpoints
    }
}


# DEBUG LOCALLY, COMMENT BEFORE COMMIT
# Invoke-NuGetPackageCaching -config "C:\src\CarbonCopy.API\nuget.config" -feedKeys @("company-shared") -nugetExePath "C:/src/nuget.exe"

## nuget.md

      
    Raw
  

              nuget.md
            
          
    Juts in case

<ItemGroup>
  	<PackageReference Include="Microsoft.Azure.Functions.Worker"
  		Version="1.22.0" />
  	<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http"
  		Version="3.2.0" />
  	<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus"
  		Version="5.20.0" />
  	<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues"
  		Version="5.4.0" />
  	<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer"
  		Version="4.3.1" />
  	<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk"
  		Version="1.17.4" />
  	<PackageReference Include="ClosedXML"
  		Version="0.100.3" />
  	<PackageReference Include="Microsoft.Azure.Functions.Extensions"
  		Version="1.1.0" />
  	<PackageReference Include="Azure.Messaging.ServiceBus"
  		Version="7.17.5" />
  	<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage"
  		Version="6.4.0" />
  	<PackageReference Include="Microsoft.Extensions.Logging.Abstractions"
  		Version="7.0.0" />
  	<PackageReference Include="Newtonsoft.Json"
  		Version="13.0.3" />
  </ItemGroup>


## usage.yml
  - task: UseDotNet@2
    displayName: 'Install .Net SDK version'
    inputs:
      packageType: sdk
      version: 8.x
      installationPath: $(Agent.ToolsDirectory)/dotnet

  - task: NuGetToolInstaller@1
    displayName: 'Use NuGet 6.9.1'
    inputs:
      versionSpec: 6.9.1


  - task: NuGetAuthenticate@1
    inputs:
      nuGetServiceConnections: 'company-shared'

      - task: PowerShell@2
      displayName: Cache nuget packages
      inputs:
        targetType: "inline"
        pwsh: true
        script: |
          $psPath = Join-Path ${env:PIPELINE_WORKSPACE} "s/.vsts/shared/Invoke-NuGetPackageCaching.ps1"
          . $psPath

          $configPath = Join-Path ${env:PIPELINE_WORKSPACE} "s/nuget.config"
          Invoke-NuGetPackageCaching -config $configPath -feedKeys @("company-shared")

	function Add-NugetAuthConfig {
	param (
	[Parameter(Mandatory = $true)]
	[string]$path,

	[Parameter(Mandatory = $true)]
	[string]$key,

	[Parameter(Mandatory = $true)]
	[string]$login,

	[Parameter(Mandatory = $true)]
	[string]$pwd
	)

	# Load the XML file
	[xml]$nugetConfig = Get-Content $path

	# Check if the packageSourceCredentials node exists, if not create it
	if ($null -eq $nugetConfig.configuration.packageSourceCredentials) {
	$packageSourceCredentialsNode = $nugetConfig.CreateElement("packageSourceCredentials")
	$nugetConfig.configuration.AppendChild($packageSourceCredentialsNode) \| Out-Null
	}
	else {
	$packageSourceCredentialsNode = $nugetConfig.configuration.packageSourceCredentials
	}

	# Create the new package source credentials node
	$newPackageSourceNode = $nugetConfig.CreateElement($key)

	$usernameNode = $nugetConfig.CreateElement("add")
	$usernameNode.SetAttribute("key", "Username")
	$usernameNode.SetAttribute("value", $login)
	$newPackageSourceNode.AppendChild($usernameNode) \| Out-Null

	$passwordNode = $nugetConfig.CreateElement("add")
	$passwordNode.SetAttribute("key", "ClearTextPassword")
	$passwordNode.SetAttribute("value", $pwd)
	$newPackageSourceNode.AppendChild($passwordNode) \| Out-Null

	# Append the new package source credentials node
	$packageSourceCredentialsNode.AppendChild($newPackageSourceNode) \| Out-Null

	# Save the updated XML back to the file
	$nugetConfig.Save($path)
	Write-Output "Updated nuget.config file saved to $path"
	}

	function Get-PasswordFromJson {
	param (
	[Parameter(Mandatory = $true)]
	[string]$JsonString,

	[Parameter(Mandatory = $true)]
	[string]$EndpointUrl
	)

	# Convert the JSON string to a PowerShell object
	$jsonObject = $JsonString \| ConvertFrom-Json

	# Iterate through the endpointCredentials array to find the matching endpoint
	foreach ($credential in $jsonObject.endpointCredentials) {
	if ($credential.endpoint -eq $EndpointUrl) {
	return $credential.password
	}
	}

	# If no matching endpoint is found, return $null
	return $null
	}

	function Invoke-PackageEnumeration {
	param (
	[string]$csProjLookupDirectory
	)

	$csProjFiles = Get-ChildItem -Path $csProjLookupDirectory -Recurse -Filter *.csproj

	# Initialize the dictionary
	$packagesDictionary = @{}

	foreach ($file in $csProjFiles) {
	[xml]$xml = Get-Content -Path $file.FullName

	$packages = $xml.Project.ItemGroup.PackageReference

	foreach ($package in $packages) {
	$packageId = $package.Include
	$packageVersion = $package.Version

	if ([string]::IsNullOrEmpty($packageId)) {
	continue;
	}

	if ($false -eq $packagesDictionary.ContainsKey($packageId) ) {
	$packagesDictionary[$packageId] = @($packageVersion)
	}
	else {
	if (-not $packagesDictionary[$packageId].Contains($packageVersion)) {
	$packagesDictionary[$packageId] += $packageVersion
	}
	}
	}
	}

	# Return the dictionary
	return $packagesDictionary
	}

	function Invoke-NuGetPackageCaching {
	param (
	[string]$config, # path to the nuget.config file, rel or abs
	[string[]]$feedKeys, # keys of the feeds to cache packages, should exist in the nuget.config file
	[string]$projectsDir = $null, # where to look for the csproj files, by default the directory of the nuget.config file
	[string]$nugetExePath = "nuget.exe" # path to the nuget.exe
	)


	$nugetConfigPath = Resolve-Path $config

	# check if projectsDir null or empty
	if ([string]::IsNullOrEmpty($projectsDir)) {
	$projectsDir = Split-Path $nugetConfigPath
	}

	if (-Not (Test-Path -Path $nugetExePath)) {
	Write-Host "Trying to resolve the nuget.exe path: '$nugetExePath'"
	$nugetPath = (Get-Command nuget.exe).Source
	Write-Host "Resolved nuget.exe path: '$nugetPath'"
	$nugetExePath = $nugetPath
	}

	Write-Warning "Prerequisites:"
	Write-Warning "nuget.exe should be installed on the machine (NuGetToolInstaller task)."
	Write-Warning "NuGetAuthenticate task should be executed before this task."

	# write the input parameters
	Write-Host "The Function Input parameters:"
	Write-Host " config: $config"
	Write-Host " feedKeys: $($feedKeys -join ', ')"
	Write-Host " nugetExePath: $nugetExePath"
	Write-Host " projectsDir: $projectsDir"

	# Get the NuGet global packages cache directory
	$globalPackagesCache = & dotnet nuget locals global-packages --list

	# Extract the path from the output
	$cachePath = $globalPackagesCache \| Select-String -Pattern "global-packages: (.+)" \| ForEach-Object { $_.Matches[0].Groups[1].Value }


	Write-Host "NuGet global packages list: $globalPackagesCache"
	Write-Host "NuGet global packages cache directory (parsed): $cachePath"

	$authProviderEnpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS
	Write-Host "NUGET_CREDENTIALPROVIDERS_PATH: $env:NUGET_CREDENTIALPROVIDERS_PATH"
	Write-Host "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS: $authProviderEnpoints"

	try {
	# VSS_NUGET_EXTERNAL_FEED_ENDPOINTS

	$env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS = ""

	# Ensure the packages folder exists
	if (-Not (Test-Path -Path $cachePath)) {
	Write-Host "Creating the NuGet global packages cache directory at '$cachePath'"
	New-Item -ItemType Directory -Path $cachePath
	}

	# ensure the nuget.config file exists, exit with error if not
	if (-Not (Test-Path -Path $nugetConfigPath)) {
	Write-Error "NuGet config file not found at the provided path '$nugetConfigPath'"
	return 1
	}

	# parse the nuget.config XML file to get all the feeds
	$cfgFeedsDic = @{}
	$xml = [xml](Get-Content $nugetConfigPath)
	$xml.configuration.packageSources.add \| ForEach-Object {
	$cfgFeedsDic[$_.key] = $_.value
	}

	# add auth info to the nuget.config file
	foreach ($key in $cfgFeedsDic.Keys) {
	$authPwd = Get-PasswordFromJson -JsonString $authProviderEnpoints -EndpointUrl $cfgFeedsDic[$key]
	if ([string]::IsNullOrEmpty($authPwd)) {
	Write-Host "The password for the feed '$key' is not found in the provided credentials. Skipping the feed..."
	continue;
	}
	Add-NugetAuthConfig -path $nugetConfigPath -key $key -login "vsts" -pwd $authPwd
	}

	# print the nuget.config file
	Write-Host "NuGet config file:"
	Get-Content $nugetConfigPath \| ForEach-Object { Write-Host $_ }

	$cfgFeeds = @()
	# print the feed URLs and keys
	Write-Host "Exctracted feeds from the provided config:"
	foreach ($key in $cfgFeedsDic.Keys) {
	Write-Host " $($key): $($cfgFeedsDic[$key])"
	$cfgFeeds += $cfgFeedsDic[$key]
	}

	$includedPackages = Invoke-PackageEnumeration -csProjLookupDirectory $projectsDir

	Write-Host "Used packages:"
	foreach ($ipKey in $includedPackages.Keys) {
	Write-Host "Package: $($ipKey):"
	foreach ($ipv in $includedPackages[$ipKey]) {
	Write-Host " - $($ipv):"
	}
	}

	foreach ($feedKey in $feedKeys) {
	$feedUrlToCache = $cfgFeedsDic[$feedKey]
	Write-Host "Getting all the packages list from the feed: '$feedKey' - '$feedUrlToCache"
	# List all packages from the feed
	if ($IsWindows -eq $true) {
	$packages = & $nugetExePath list -Source $feedUrlToCache -ConfigFile $nugetConfigPath -NonInteractive -verbosity detailed
	}
	else {
	$packages = mono $nugetExePath list -Source $feedUrlToCache -ConfigFile $nugetConfigPath -NonInteractive -verbosity detailed
	}

	Write-Host "Caching packages from the feed: '$feedKey"

	foreach ($package in $packages) {
	Write-Host "Processing package: '$package'"
	if ($package -match "^(\S+)\s(\S+)$") {
	$packageName = $matches[1]
	$packageVersions = @($matches[2])

	# if we dont have this package in the used one, we skip it

	if ($false -eq $includedPackages.ContainsKey($packageName)) {
	Write-Host "!!! The package '$packageName' not found in the included packages! Skipping it from caching..."
	continue;
	}

	$includedVersions = $includedPackages[$packageName]

	if ($includedVersions.count -gt 0) {
	$packageVersions = $includedVersions
	}

	# caching all the listed package versions
	foreach ($packageVersion in $packageVersions) {
	Write-Host "Downloading package '$packageName.$packageVersion'"

	if ($IsWindows -eq $true) {
	& $nugetExePath install $packageName -Version $packageVersion -OutputDirectory $cachePath -ConfigFile $nugetConfigPath -NonInteractive -MSBuildPath $msbuildPath -verbosity detailed #-Source $cfgFeeds
	}
	else {
	mono $nugetExePath install $packageName -Version $packageVersion -OutputDirectory $cachePath -ConfigFile $nugetConfigPath -NonInteractive -MSBuildPath $msbuildPath -verbosity detailed #-Source $cfgFeeds
	}
	}
	}
	}
	}
	}
	catch {
	Write-Error $_.Exception.Message
	return 1
	}
	finally {
	$env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS = $authProviderEnpoints
	}
	}




	# DEBUG LOCALLY, COMMENT BEFORE COMMIT
	# Invoke-NuGetPackageCaching -config "C:\src\CarbonCopy.API\nuget.config" -feedKeys @("company-shared") -nugetExePath "C:/src/nuget.exe"
	- task: UseDotNet@2
	displayName: 'Install .Net SDK version'
	inputs:
	packageType: sdk
	version: 8.x
	installationPath: $(Agent.ToolsDirectory)/dotnet

	- task: NuGetToolInstaller@1
	displayName: 'Use NuGet 6.9.1'
	inputs:
	versionSpec: 6.9.1


	- task: NuGetAuthenticate@1
	inputs:
	nuGetServiceConnections: 'company-shared'

	- task: PowerShell@2
	displayName: Cache nuget packages
	inputs:
	targetType: "inline"
	pwsh: true
	script: \|
	$psPath = Join-Path ${env:PIPELINE_WORKSPACE} "s/.vsts/shared/Invoke-NuGetPackageCaching.ps1"
	. $psPath

	$configPath = Join-Path ${env:PIPELINE_WORKSPACE} "s/nuget.config"
	Invoke-NuGetPackageCaching -config $configPath -feedKeys @("company-shared")