test.groovy

/**
 * Makes a tester closure that runs some tests and reports back the result.
 */
def make_a_tester(String label, String stash_name,
                  String constraint_stash_name,
                  String venv_py_version, String test_path,
                  String report_name, Integer test_timeout=3600) {
    tester = {
        node(label) {
            timeout(time: test_timeout, unit: 'SECONDS') {
                wrap([$class: 'AnsiColorBuildWrapper', colorMapName: 'xterm']) {
                    clean_ws {
                        // Get the stash!
                        unstash(stash_name)
                        // Get the constraint stash.
                        sh "mkdir -p .reports"
                        dir(".reports") {
                            unstash(constraint_stash_name)
                        }
                        try {
                            named_sh("Create a venv",
                            """
                            virtualenv --python=${venv_py_version} .venv
                            """)
                            named_sh("Setup the venv",
                            """
                                source .venv/bin/activate
                                # The built-in versions of these are usually so old that
                                # they just will not work so force them to be upgraded...
                                pip install setuptools pip --upgrade
                                # Install all the things!
                                pip install -r requirements.txt \
                                            -r test-requirements.txt \
                                            -c .reports/upper-constraints.txt
                                # This will make sure any project entrypoints
                                # are registered (which may or may not be used by
                                # tests).
                                pip install .
                                # Extra testing requirements.
                                #
                                # Needed so we can convert tox, testr, subunit results into
                                # a format that jenkins understands so that we can
                                # post them after the testing is done (and have them easily
                                # viewable...)
                                pip install os-testr \
                                            -c .reports/upper-constraints.txt
                                # Include these in files that show up on jenkins...
                                pip freeze > .reports/freeze.txt
                            """)
                            named_sh("List and run the tests",
                            """
                                source .venv/bin/activate
                                # Taken from nova tox.ini
                                export OS_STDOUT_CAPTURE=1
                                export OS_STDERR_CAPTURE=1
                                export LANGUAGE=en_US
                                export LC_ALL=en_US.utf-8
                                # We may have to change this for different
                                # projects...
                                export OS_TEST_PATH=${test_path}
                                export OS_TEST_TIMEOUT=${test_timeout}
                                testr init
                                # Ensure tests list; otherwise this is going
                                # to end badly.
                                echo "Listing tests, please wait..."
                                testr list-tests > .reports/tests.txt
                                echo "Running tests, please wait..."
                                # We will handle having failing tests later...
                                set +e
                                testr run --load-list .reports/tests.txt \
                                          --subunit | subunit-trace -f --color
                            """)
                            named_sh("Process test results",
                            """
                                source .venv/bin/activate
                                # This may have issues if a project is not using
                                # subunit or testr...
                                #
                                # Pretty much everything in openstack is, so we do not yet
                                # need to worry about that; ie the usage of only `nosetests`
                                # or similar is minimal...
                                testr last --subunit > .reports/results.subunit
                                cd .reports/
                                # Avoid dying on these, because we may already be in a fail state
                                # if the tests failed and we just want to make the output files
                                # for uploading...
                                set +e
                                # Do all our output conversions (so that people can
                                # easily get at the results in jenkins).
                                subunit2html results.subunit results.html
                                cat results.subunit | subunit-trace -n > results.txt
                                if [ \$? == 0 ]; then
                                    date --utc > passed
                                fi
                            """)
                        }
                        finally {
                            // Always try to upload something...
                            publishHTML(target: [allowMissing: true,
                                                 alwaysLinkToLastBuild: false,
                                                 keepAll: true,
                                                 reportDir: ".reports",
                                                 reportFiles: "*.*",
                                                 reportName: report_name])
                        }
                        // Figure out if it failed or not...
                        if(fileExists(".reports/passed")) {
                            has_failed = false
                        }
                        else {
                            has_failed = true
                        }
                        // Just incase the above shell script didn't work, if it
                        // failed, then no file will be here...
                        if(fileExists(".reports/results.txt")) {
                            results = readFile(".reports/results.txt")
                            result_summary = extract_testr_summary(results)
                        }
                        else {
                            results = ""
                            result_summary = ""
                        }
                        if(result_summary) {
                            msg = msg_prefix + msg_postfix + "\n"
                            msg += "```\n"
                            msg += result_summary
                            msg += "\n```\n"
                            if(has_failed) {
                                slackSend(message: msg, failOnError: false,
                                          color: "warning")
                            }
                            else {
                                slackSend(message: msg, failOnError: false,
                                          color: "good")
                            }
                        }
                        if(has_failed) {
                            if(result_summary) {
                                throw new Exception("Testing did not succeed: ${result_summary}")
                            }
                            else {
                                throw new Exception("Testing did not succeed")
                            }
                        }
                    }
                }
            }
        }
    }
    return tester
}