Cypress e2e and Jenkinsfile: Retries the failed e2e spec files. Passes the stage (don't fail) if all test cases that failed the first time pass the second time. Ignores test cases that fail the second time that passed the first time.

index.js

// ***********************************************************
// This example plugins/index.js can be used to load plugins
//
// You can change the location of this file or turn off loading
// the plugins file with the 'pluginsFile' configuration option.
//
// You can read more here:
// https://on.cypress.io/plugins-guide
// ***********************************************************

const { intersection } = require('lodash');
const glob = require('glob');
const { writeFileSync, existsSync, readFileSync, unlinkSync } = require('fs');
const { join } = require('path');

const repoRootPath = join(__dirname, '..', '..', '..', '..');

module.exports = (on, config) => {
  // Your other Cypress plugins here

  // When running non-interactively and not running storybook specs, this saves
  // spec files that the Jenkinsfile should retry, and saves the test cases that
  // have not passed yet. Saving the test cases allows the Jenkinsfile to not
  // fail the e2e stage if e.g. the individual tests that failed the first time
  // passed the second time, and to not fail if tests that passed the first time
  // failed the second time:
  on('after:run', (results) => {
    if (
      results &&
      results.runs.every((r) => !r.spec.relative.includes('storybook'))
    ) {
      const failedSpecs = results.runs.filter((r) => r.stats.failures !== 0);
      let failedTests = [];
      for (const run of failedSpecs) {
        for (const test of run.tests) {
          if (test.state === 'failed') {
            failedTests.push(
              JSON.stringify([run.spec.relative, ...test.title]),
            );
          }
        }
      }
      const appE2ePath = join(repoRootPath, 'tmp', 'app-e2e');
      const failedTestsPath = join(appE2ePath, 'tests-still-failing.txt');
      if (existsSync(failedTestsPath)) {
        const prevFailedTests = readFileSync(failedTestsPath, {
          encoding: 'utf8',
        });
        failedTests = intersection(prevFailedTests.split('\n'), failedTests);
      }
      const specsToRetry = failedSpecs.map((run) =>
        join('**', run.spec.relative),
      );
      const specsToRetryPath = join(appE2ePath, 'specs-to-retry.txt');
      writeFileSync(specsToRetryPath, specsToRetry.join(','));
      writeFileSync(failedTestsPath, failedTests.join('\n'));

      if (failedTests.length === 0) {
        // This part deletes mocha files; modify if your config generates a
        // different kind of report files or just delete if you don't use
        // reporting:

        // If all tests passed at least once, delete the non-storybook mocha
        // results files; the tests passed anyway and deleting the files is the
        // only way without parsing the XML to prevent Jenkins from setting the
        // pipeline to "unstable" due to failures it finds in these files:
        const testResultsPath = join(
          repoRootPath,
          'build',
          'test-results',
          'e2e-*.xml',
        );
        for (const e2eMochaFilepath of glob.sync(testResultsPath)) {
          const contents = readFileSync(e2eMochaFilepath, { encoding: 'utf8' });
          if (!contents.includes('storybook')) {
            unlinkSync(e2eMochaFilepath);
          }
        }
      }
    }
  });

  return config;
};

Jenkinsfile

// The stage for e2e in the Jenkinsfile:
stage('E2E') {
  steps {
    script {
      nodejs(your_jenkins_nodejs_plugin_config_options_here) {
        try {
          sh 'yarn start-server-and-script e2e:prod'
        } catch (err) {
          def specsToRetryFilePath = 'tmp/app-e2e/specs-to-retry.txt';
          // Don't retry if file doesn't exist which happens
          // when e2e stage was cancelled due to a different
          // parallel stage failing:
          if (fileExists(specsToRetryFilePath)) {
            // If e2e failed and app-e2e/src/plugins/index.js
            // saved specs to retry, re-run just those specs:
            def specsToRetry = readFile(specsToRetryFilePath);
            if (specsToRetry.trim() != '') {
              try {
                // Run the command again, specifying which spec files to retry.
                // The frontend should not take long to rebuild because the Nx
                // cache should be able to provide it without actually building
                // it again:
                sh "SPEC=${specsToRetry} yarn start-server-and-script e2e:prod:spec"
              } catch (retryErrors) {
                def testsStillFailing = readFile('tmp/app-e2e/tests-still-failing.txt')
                // Don't error if e2e failed again but the
                // individual test _cases_ that failed in the
                // first run passed in the second run:
                if (testsStillFailing.trim() != '') {
                  throw retryErrors;
                }
              }
            } else {
              throw err;
            }
          }
        }
      }
    }
  }
}

package.json

/* This is some relevant parts of the package.json */
{
  "scripts": {
    "nx": "nx",
    "server:ci": "node -r dotenv/config dist/apps/server/main.js",
    "e2e:prod": "nx e2e app-e2e --configuration=production",
    "e2e:prod:spec": "yarn e2e:prod --spec $SPEC",
    /* `server:ci http://localhost:3334` assumes your app has one API server */
    /* or gateway to start locally which is on port 3334: */
    "start-server-and-script": "start-server-and-test server:ci http://localhost:3334",
  },
  "devDependencies": {
    "@nrwl/cli": "13.10.1",
    "@nrwl/cypress": "13.10.1",
    "cypress": "9.5.4",
    "glob": "8.0.3",
    "jest": "27.2.3",
    "start-server-and-test": "1.14.0"
  }
}

why.md

Why not just use Cypress's retry in the spec.ts files or config.json?

You can put retry inside the spec.ts files or in the config.json to use Cypress's built in retry mechanism. The problem is very often we see test cases fail the first time and all subsequent retries, but when re-running the e2e command they pass. This solves that by re-running the e2e command on the spec files that failed. Also, putting retry(2) in Cypress's config.json doubles the time the e2e stage takes in the pipeline if most of the tests are broken or otherwise failing.

So why not just use Jenkins's retry() in the Jenkinsfile to re-run the e2e command?

Re-running all the tests if there is a failure will make the stage take twice as long. On top of that, if any tests that passed the first run fail in the second run, then the stage will fail, whereas we want the e2e stage to pass even if some tests passed in only one of the runs.

Why re-run the failed specs and not just the failed tests?

Nrwl Nx's e2e command, and perhaps Cypress's own commands, only provide a filter for spec files. There is a package called cypress-grep that could be used. It could be a nice improvement, although it might make this gist even more complex.