Created
September 24, 2022 19:53
Star
You must be signed in to star a gist
Using Nx Affected in Azure Pipelines
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* This script decides whether containers need to be re-built (and thus re-deployed). | |
* The result is a special `task.setvariable` printed to the console for affected containers, | |
* to be picked up by the next build stage in the pipeline. | |
* | |
* Usage: | |
* | |
* node is-affected.js --pipelineId 1 --app website --app dashboard --app services | |
* | |
* Example output: | |
* | |
* ##vso[task.setvariable variable=BUILD_WEBSITE;isOutput=true]AFFECTED | |
* ##[warning]website is affected (base: d9cea66b57a4704f6665a82461d14f92f927bfd8) | |
* ##vso[task.setvariable variable=BUILD_DASHBOARD;isOutput=true]TAG_NOT_FOUND | |
* ##[warning]dashboard was not previously tagged (base: d9cea66b57a4704f6665a82461d14f92f927bfd8) | |
* ##[warning]services is NOT affected (base: d9cea66b57a4704f6665a82461d14f92f927bfd8) | |
* | |
* Implementation notes: | |
* | |
* - Tags: | |
* - The tag is equal to the container name for Nx project type "app" (e.g. `website`). | |
* - The commit sha from the queried tag represents the latest successful build for that container. | |
* - A tag is added to the pipeline run only after a successful build of any container separately (look for "Tag successful build") | |
* - Calculate whether container is affected between HEAD and this commit sha. | |
* - If AFFECTED, echo special `task.setvariable` syntax for next build stage to use in its run conditions. | |
* - Next build stage reads special variable syntax to build or not (look for "Build ${{ parameters.containerName }}") | |
* - If TAG_NOT_FOUND, also set this variable and build container. | |
* - The `az pipelines runs list` command requires a PAT (personal access token or `AZURE_DEVOPS_TOKEN`). | |
* | |
* Related Azure docs: | |
* | |
* - https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#set-variables-in-scripts | |
* - https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#use-output-variables-from-tasks | |
*/ | |
const util = require('node:util'); | |
const execFile = util.promisify(require('node:child_process').execFile); | |
const { | |
values: { pipelineId, debug = false, app = [], lib = [] }, | |
} = util.parseArgs({ | |
options: { | |
debug: { type: 'boolean' }, | |
pipelineId: { type: 'string' }, | |
app: { type: 'string', multiple: true }, | |
lib: { type: 'string', multiple: true }, | |
}, | |
}); | |
const isDebug = debug || process.env.SYSTEM_DEBUG === 'True'; | |
const AffectedCache = { | |
app: new Map(), | |
lib: new Map(), | |
}; | |
const writeVariable = (key, value) => console.log(`##vso[task.setvariable variable=${key};isOutput=true]${value}`); | |
const logWarning = message => console.log(`##[warning]${message}`); | |
const logError = message => console.log(`##[error]${message}`); | |
const logDebug = (...args) => isDebug && console.debug(...args); | |
/** | |
* @param {string} containerName | |
* @returns {string} | |
*/ | |
const getVariableName = containerName => `BUILD_${containerName.toUpperCase().replace(/-/g, '_')}`; | |
/** | |
* Execute command in child process, and output stdout/stderr in debug mode | |
* | |
* @param {string} program | |
* @param {string[]} args | |
* @returns {void} | |
*/ | |
const exec = async (program, args) => { | |
logDebug(program, args.join(' ')); | |
const { stdout, stderr } = await execFile(program, args); | |
logDebug({ stdout, stderr }); | |
if (stderr) throw new Error(stderr); | |
if (stdout) return stdout.trim(); | |
}; | |
/** | |
* Find pipeline run of latest successful build for tag (= containerName), and return the associated commit sha | |
* Ideally we would use `az pipelines` programmatically here (i.e. not in a child process), but not available atm? | |
* | |
* @param {string} containerName | |
* @returns {string} | |
*/ | |
const getLatestBuildCommit = async containerName => { | |
const args = [ | |
'pipelines', | |
'runs', | |
'list', | |
['--branch', 'main'], | |
['--pipeline-ids', pipelineId], | |
['--tags', containerName], | |
['--query-order', 'StartTimeDesc'], // `FinishTimeDesc` gives second latest? | |
['--query', '[].[sourceVersion]'], | |
['--top', 1], | |
['--out', 'tsv'], | |
].flat(); | |
return await exec('az', args); | |
}; | |
/** | |
* Find affected Nx projects since `base` commit sha | |
* Ideally we would use `nx print-affected` programmatically here (i.e. not in a child process), but not available atm? | |
* | |
* @param {string} sha | |
* @param {"app" | "lib"} type | |
* @returns {string[]} | |
*/ | |
const getAffectedProjectsSinceCommit = async (sha, type) => { | |
const args = [ | |
'print-affected', | |
['--type', type], | |
['--select', 'projects'], | |
['--base', sha], | |
['--head', 'HEAD'], | |
].flat(); | |
const stdout = await exec('nx', args); | |
return stdout.split(', '); | |
}; | |
/** | |
* @param {string} nxProject | |
* @param {"app" | "lib"} type | |
* @param {string} sha | |
* @returns {boolean} | |
*/ | |
const isProjectAffected = async (nxProject, type, sha) => { | |
if (AffectedCache[type].has(sha)) { | |
// Available from cache, nothing to do | |
} else if (sha.match(/^[0-9a-f]{5,40}$/)) { | |
const affected = await getAffectedProjectsSinceCommit(sha, type); | |
AffectedCache[type].set(sha, affected); | |
} | |
const affectedContainersForCommit = AffectedCache[type].get(sha); | |
return affectedContainersForCommit.includes(nxProject); | |
}; | |
/** | |
* @param {"app" | "lib"} type | |
* @param {string} nxProject | |
* @param {string} libContainerName | |
* @returns {void} | |
*/ | |
const printAffectedProjects = async (type, nxProject, libContainerName) => { | |
try { | |
const containerName = libContainerName ?? nxProject; | |
const variableName = getVariableName(containerName); | |
const sha = await getLatestBuildCommit(containerName); | |
if (sha) { | |
const isAffected = await isProjectAffected(nxProject, type, sha); | |
if (isAffected) { | |
writeVariable(variableName, 'AFFECTED'); | |
logWarning(`${nxProject} is affected (base: ${sha})`); | |
} else { | |
logWarning(`${nxProject} is NOT affected (base: ${sha})`); | |
} | |
} else { | |
writeVariable(variableName, 'TAG_NOT_FOUND'); | |
logWarning(`${nxProject} was not previously tagged`); | |
} | |
} catch (error) { | |
logError(`${nxProject} query gave an error: ${error.toString()}`); | |
console.error(error); | |
process.exit(1); | |
} | |
}; | |
const main = async () => { | |
for (const nxProject of app) await printAffectedProjects('app', nxProject); | |
for (const nxProject of lib) await printAffectedProjects('lib', ...nxProject.split(',')); | |
if (isDebug) console.debug(AffectedCache); | |
}; | |
main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Usage: is-affected [lib|app] [nx-project-name] [BUILD_NX_PROJECT_NAME] | |
is-affected() { | |
local SHA=$(az pipelines runs list --branch main --pipeline-ids $(System.DefinitionId) --tags "$2" --query-order FinishTimeDesc --query '[].[sourceVersion]' --top 1 --out tsv) | |
local WRITE_VARIABLE="##vso[task.setvariable variable=$3;isOutput=true]"; | |
if [[ "$SHA" =~ [0-9a-f]{5,40} ]]; then | |
local AFFECTED=$(npx nx print-affected --type=${1} --select=projects --plain --base=$SHA --head=HEAD) | |
if [[ "$AFFECTED" == *"$2"* ]]; then | |
echo "${WRITE_VARIABLE}AFFECTED" | |
echo "##[warning]$2 is affected (base: $SHA)" | |
else | |
echo "##[warning]$2 is NOT affected (base: $SHA)" | |
fi | |
elif [[ -z "$SHA" ]]; then | |
echo "${WRITE_VARIABLE}TAG_NOT_FOUND" | |
echo "##[warning]$2 was not previously tagged" | |
else | |
echo "##[error]$2 query gave an error: $SHA" | |
fi | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
stages: | |
- stage: Prepare | |
pool: | |
vmImage: ubuntu-latest | |
jobs: | |
- job: Determine_Affected | |
displayName: Determine Affected Nx Projects | |
steps: | |
- task: NodeTool@0 | |
displayName: Use Node.js v16.14.1 | |
inputs: | |
versionSpec: 16.14.1 | |
- script: npm install nx | |
displayName: Install Nx | |
# The Azure CLI tool may require you to set the `organization` and `project` first | |
- bash: | | |
az config set extension.use_dynamic_install=yes_without_prompt | |
az devops configure --defaults organization=$(System.TeamFoundationCollectionUri) project="$(System.TeamProject)" | |
displayName: Set default Azure DevOps organization and project | |
- bash: | | |
is-affected() { | |
local SHA=$(az pipelines runs list ...) | |
... | |
} | |
is-affected app my-app BUILD_MY_APP | |
name: AffectedNxProjects | |
displayName: Determine affected Nx projects | |
env: | |
AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
- script: > | |
node is-affected.js | |
--pipelineId $(System.DefinitionId) | |
--app website | |
--app dashboard | |
--app services | |
name: AffectedProjects | |
displayName: Determine affected projects | |
env: | |
AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment