Azure DevOps Pipeline Maintenance Jobs to update NVD DB on multiple agents

schedulerPipeline.yml

# This is the pipeline that runs any scheduled maintainance jobs
# we wish to run in addition to the built in Azure DevOps Maintainance jobs
name: "Update NVD Database on $(agent)"

# The parameters to target each pool and agent
parameters:
  - name: pool
    type: string
  - name: agent
    type: string
  - name: purge
    type: boolean
    default: false
  - name: useAPIKey
    type: boolean
    default: true

# We cannot use to the parameters directly else we get a 'A template expression is not allowed in this context'
# However, if we alias them with a variable they work
variables:
  - name: pool
    value: ${{parameters.pool}}
  - name: agent
    value: ${{parameters.agent}}
  - name: purge
    value: ${{parameters.purge}}
  - name: useAPIKey
    value: ${{parameters.useAPIKey}}
  - name: BuildDefintion 
    value: "UpdateNVDDBFromBuildArtifact"
  # filter to limit the scope of agent pools to consider
  - name: PoolNamePrefix
    value: "POOL-" # your prefix if any
    
trigger: none

schedules:
- cron: '0 0 * * *'
  displayName: Daily midnight build
  always: true
  branches:
    include:
    - main

jobs:
- job: NVDUpdate
  pool:
    name: "$(pool)"
    demands: Agent.Name -equals $(agent)
  timeoutInMinutes: 180 
  steps:
  # Need to set the verison of JAVA as the underlying tool
  # does not support the default in the agent 
  - task: JavaToolInstaller@0
    inputs:
      versionSpec: '17'
      jdkArchitectureOption: 'x64'
      jdkSourceOption: 'PreInstalled'

  # find the local NVD location
  - powershell: |
      $nvdcachepath = $(get-childitem "$(Agent.WorkFolder)\_tasks\dependency-check-build-task*\*.*.*\dependency-check\data" | Sort-Object LastWriteTime -Descending | Select-Object -First 1).FullName
      write-Host "##vso[task.setvariable variable=nvdcachepath;]$nvdcachepath"
    displayName: Find NVD DB path

  # we delete the old version of the DB as there is potential a schema change on the auto downloaded DB
  - powershell: |
      Remove-Item -LiteralPath $(nvdcachepath) -Force -Recurse
    displayName: Delete contents of NVD DB
    condition: and(succeeded(), eq(variables.purge, true))

  # The run the task to populate the cache
  - task: dependency-check-build-task@6
    displayName: "Update the Vunerability Scan DB"
    inputs:
      projectName: 'Maintainance'
      scanPath: '.'
      format: 'HTML'
      ${{ if eq(variables.useAPIKey, true)}}:
        additionalArguments: '--nvdApiKey $(nvdapikey) --updateonly --nvdApiDelay 6000 --nvdMaxRetryCount 20' 
      ${{ if eq(variables.useAPIKey, false)}}:
        additionalArguments: ' --updateonly --nvdApiDelay 6000 --nvdMaxRetryCount 20' 
  
  - task: PublishPipelineArtifact@1
    inputs:
      targetPath: '$(nvdcachepath)'
      artifact: 'NVD-DB'
      publishLocation: 'pipeline'

  - task: AzureCLI@2
    inputs:
      # The Azure Subscription is the link to a Service Principle
      # with permission to access the agent pools and queue builds
      azureSubscription: 'ScheduledMaintainanceJobs'
      scriptType: 'pscore'
      scriptLocation: 'inlineScript'
      inlineScript: |
        write-host "Find the agent pools with the prefix '$(PoolNamePrefix)'"
        $Pools = $(az pipelines pool list --organization $(System.TeamFoundationCollectionUri) --query "[? (starts_with(name,'$(PoolNamePrefix)'))].[id,name]" --output tsv)
        write-host "$($Pools.count) pools found"
        foreach ($pool in $pools) {
            # ugly but works with tsv format data
            $poolSplit = $pool.Split("`t")
            $poolID = $poolSplit[0]
            $poolName = $poolSplit[1]

            write-host "Find the agents the pool '$poolName'"

            $Agents = az pipelines agent list --organization $(System.TeamFoundationCollectionUri) --pool-id $PoolID --query "[?enabled].name"  --output tsv 
            
            # trigger an update on all agents other than the one running this pipeline
            foreach ($Agent in $Agents | where-object { $_ -ne "$(agent)"}) {
                $buildNameid = $(az pipelines run --organization $(System.TeamFoundationCollectionUri) --name $(BuildDefintion) --project $(System.TeamProject) --parameters "pool=$PoolName" "agent=$Agent" "pipelineId=$(Build.BuildId)" "purge=${{parameters.purge}}" --query "name"  --output tsv )
                Write-host "Queued build $buildNameid "
            }

        }
    displayName: 'Trigger maintainance builds on all active agents'

UpdateNVDDBFromBuildArtifact.yml

# This is the pipeline that runs any scheduled maintainance jobs
# we wish to run in addition to the built in Azure DevOps Maintainance jobs
name: "Update NVD Database on $(agent)"

# The parameters to target each pool and agent
parameters:
  - name: pool
    type: string
    default: ''
  - name: agent
    type: string
    default: ''
  - name: project
    type: string
    default: 'BM'
  - name: definition
    type: string
    default: '123' # set the ID of schedulerPipeline
  - name: artifactName
    type: string
    default: 'NVD-DB' 
  - name: pipelineId
    type: string
  - name: purge
    type: boolean
    default: false

# We cannot use to the parameters directly else we get a 'A template expression is not allowed in this context'
# However, if we alias them with a variable they work
variables:
  - name: pool
    value: ${{parameters.pool}}
  - name: agent
    value: ${{parameters.agent}}
  - name: project
    value: ${{parameters.project}}
  - name: definition
    value: ${{parameters.definition}}    
  - name: artifactName
    value: ${{parameters.artifactName}}    
  - name: pipelineId
    value: ${{parameters.pipelineId}}    
  - name: purge
    value: ${{parameters.purge}}
    
trigger: none

jobs:
- job: NVDUpdate
  pool:
    name: "$(pool)"
    demands: Agent.Name -equals $(agent)

  steps:
     - checkout: none

    - powershell: |
        $nvdcachepath = $(get-childitem "$(Agent.WorkFolder)\_tasks\dependency-check-build-task*\*.*.*\dependency-check\data" | Sort-Object LastWriteTime -Descending | Select-Object -First 1).FullName
        write-host "Using NVD Path of $nvdcachepath"
        write-Host "##vso[task.setvariable variable=nvdcachepath;]$nvdcachepath"
      displayName: Find NVD DB path

    - task: DownloadPipelineArtifact@2
      displayName: Download NVD DB
      inputs:
        buildType: 'specific'
        project: '$(project)'
        definition: '$(definition)'
        buildVersionToDownload: 'specific'
        pipelineId: '$(pipelineId)'
        artifactName: '$(artifactName)'
        targetPath: '$(Pipeline.Workspace)\nvddata'  # we could in theory download direct to target folder

    # we delete the old version of the DB as there is potential a schema change on the auto downloaded DB
    - powershell: |
        write-host "Deleting all the files in the $(nvdcachepath)"
        Remove-Item -LiteralPath $(nvdcachepath) -Force -Recurse
      displayName: Find the NVD Cache path and delete the contents
      condition: and(succeeded(), eq(variables.purge, true))

    - powershell: |
        write-host "Copying '$(Pipeline.Workspace)\nvddata' files to '$(nvdcachepath)'"
        copy-item -path '$(Pipeline.Workspace)\nvddata\*.*' -destination '$(nvdcachepath)' -recurse -force
      displayName: Update NVD DB Files