Test that `npm run <script>` respects exit codes

npm_run_exit_code_check.js

// The purpose of this script is to check if `npm run <user-script>` will exit
// with the proper exit code if the script fails. We cannot trust anything if
// exit codes are not respected. I mean... just imagine...
//
//     npm run test && npm run deploy
//
// See the following to understand this level of paranoia:
//
// https://github.com/npm/npm/issues/14339
// https://github.com/npm/cli/issues/6399
// https://github.com/npm/cli/issues/6399#issuecomment-1865365079
//
// Since this script detects a fundamental flaw with `npm run`, it will actually
// break all of your repo's package.json files if it discovers `npm run` does
// not respect exit codes. It does this using a syntax error at the top of the
// file to prevent `npm run <script>` from working at all. It is highly
// recommended that you have your package.json files under version control
// before running this script.
//
// This script is intentionally written with no dependencies other than Node,
// NPM, and a shell that allows `&&` between commands as control flow.
//
// To use this script, copy it into your project somewhere and then run it with
// from the root of your repo with `node path/to/npm_run_exit_code_check.js`. On
// my team at work, we force this command to be run every time we `npm install`,
// depending on if our version of NPM has changed.
//
//    "prepare": "node npm_version_changed.js && node scripts/npm_run_exit_code_check.js"
//
// `npm_version_changed.js` is left as an exercise to the reader because I don't
// know your setup. We use ASDF for version management on my team, so
// `npm_version_changed.js` just checks if the `.tool-versions` file has changed
// since the main branch.
//
// Author: @happycollision (github)  
// Gist: https://gist.github.com/happycollision/445c5e3ff6fb6c6eaa1161a441bb7553
//
// Feel free to copy/paste/change as needed. Please leave my github handle and
// link to the original Gist in here for this one reason: If this script was
// helpful to you, let me know!

const fs = require("fs")
const path = require("path")
const { execSync } = require("child_process")

const npmVersion = execSync("npm --version", { encoding: "utf8" }).trim()

const EXPECTED_EXIT_CODE = 23

const testScriptName = "failTest"
const testScriptCmd = `exit ${EXPECTED_EXIT_CODE}`

/** @typedef {{fullPath:string, relativePath:string, exitCode:number}} Result */

/**
 * @type {Result[]}
 */
const warnings = []

/**
 * @type {Result[]}
 */
const failures = []

/**
 * @param {string|number} str
 */
function red(str) {
	return `\u001B[31m${str}\u001B[0m`
}

/**
 * @param {string|number} str
 */
function yellow(str) {
	return `\u001B[33m${str}\u001B[0m`
}

/**
 * @param {string|number} str
 */
function green(str) {
	return `\u001B[32m${str}\u001B[0m`
}

/**
 * @param {string|number} str
 */
function blue(str) {
	return `\u001B[34m${str}\u001B[0m`
}

/**
 * @param {string} filename
 * @param {string} data
 */
function prependToFile(filename, data) {
	const currentContent = fs.readFileSync(filename, "utf8")
	fs.writeFileSync(filename, data + currentContent)
}

/**
 * @param {string} dir
 * @param {string[]} foundFiles
 */
function findPackageJsonFiles(dir, foundFiles = []) {
	const files = fs.readdirSync(dir, { withFileTypes: true })
	const dirs = []

	for (const file of files) {
		const filePath = path.join(dir, file.name)
		if (file.isDirectory() && file.name !== "node_modules") {
			dirs.push(filePath)
		} else if (file.isFile() && file.name === "package.json") {
			foundFiles.push(filePath)
		}
	}

	dirs.forEach((d) => findPackageJsonFiles(d, foundFiles))

	return foundFiles
}

/**
 * @param {string} packageJsonPath
 */
function testPackageJsonFile(packageJsonPath) {
	const initialPackageJsonContent = fs.readFileSync(packageJsonPath, "utf8")
	const packageJsonData = JSON.parse(initialPackageJsonContent)

	const relativePathToPackageJsonFile = path.relative(
		process.cwd(),
		packageJsonPath
	)
	const relativePathToPackageJsonDirectory = path.dirname(
		relativePathToPackageJsonFile
	)

	// Add a failing script to package.json
	packageJsonData.scripts = packageJsonData.scripts || {}
	packageJsonData.scripts[testScriptName] = testScriptCmd
	fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonData, null, 2))

	// run the script then check the exit code it produces
	let receivedExitCode = 0
	console.log(`\t${relativePathToPackageJsonFile}`)

	try {
		execSync(
			`cd ${relativePathToPackageJsonDirectory} && npm run ${testScriptName}`,
			{ stdio: "ignore" }
		)
		receivedExitCode = 0
	} catch (error) {
		// @ts-ignore
		receivedExitCode = Number(error.status)
	} finally {
		fs.writeFileSync(packageJsonPath, initialPackageJsonContent, "utf8")
	}

	// Report the results and capture for final message.
	const result = {
		fullPath: packageJsonPath,
		relativePath: relativePathToPackageJsonFile,
		exitCode: receivedExitCode,
	}

	if (receivedExitCode < 1) {
		failures.push(result)
		return console.log(red("\t\tFailed."))
	}

	if (receivedExitCode !== EXPECTED_EXIT_CODE) {
		warnings.push(result)
		return console.log(
			yellow(
				`\t\tWarning: Exit code was ${receivedExitCode} instead of ${EXPECTED_EXIT_CODE}.`
			)
		)
	}

	console.log(green("\t\tSuccess!"))
}

function main() {
	console.log(
		"Testing that `npm run <user-script>` correctly exits with a non-zero exit code when a script fails..."
	)

	findPackageJsonFiles(".").map((packageJsonPath) =>
		testPackageJsonFile(path.resolve(packageJsonPath))
	)

	if (warnings.length > 0) {
		console.log(
			`\n\nNOTE: Some package.json files above had an ${yellow("unexpected exit code")}, but still exited with a non-zero exit code. This is a warning, not a failure, since it still enables us to use control flow with \`${blue("npm run <user-script> && some-other-cmd")}\`.`
		)
	}

	if (failures.length > 0) {
		for (const x of failures) {
			prependToFile(
				x.fullPath,
				"----intentional syntax error to prevent futher use---\n"
			)
		}

		let errorMsg = `
Your version of NPM (${npmVersion}) does not correctly exit \`${blue("npm run <user-script>")}\` with non-zero exit codes if they fail. This means we cannot safely rely on any scripts that use \`${blue("npm run <some-script> && some-other-cmd")}\` as control flow.

${red("The following package.json files were intentionally broken to prevent use:")}

${failures.map((x) => `\n\t ${blue(x.relativePath)}\n\t\texit code: ${yellow(x.exitCode)}`).join("")}

The package.json files above were altered by adding the following script:

\t"scripts": {
\t\t${green(`"${testScriptName}": "${testScriptCmd}",`)}
\t\t...
\t}

Then each package.json file was tested by running the following command:

\t${green(`npm run ${testScriptName}`)}

If the exit code was not greater than 0, then the package.json file was intentionally broken to prevent further use.

Please revert the changes made to these package.json files and then correct this issue, most likely by changing your version of NPM.

${red("Failure. See above.")}
`

		console.error(errorMsg)
		process.exit(1)
	}
}

main()