QUnit runner using PhantomJS.js

qunit-runner.js

//
// Runs a folder of tests or a single test, using QUnit and outputs a JUnit xml file ready for sweet integration with your CI server.
//
// @usage
// DISPLAY=:0 phantomjs qunit-runner.js --qunit qunit.js --tests tests-foler/ --package package.json --junit qunit-results.xml
//
// package.json is a common JS package file which contains a list of files that need to be loaded before the tests are run. In this instance
// the package.json lists the files for a JS SDK, that gets loaded and then the tests test the SDK.
//
// Designed to run via PhantomJS.
//

//
// Runs a folder of tests or a single test, using QUnit and outputs a JUnit xml file ready for sweet integration with your CI server.
//
// Designed to run via PhantomJS.
//
//
// Runs a folder of tests or a single test, using QUnit and outputs a JUnit xml file ready for sweet integration with your CI server.
//
// Designed to run via PhantomJS.
//

var QUnitRunner = function (args) {
	// Take the args array and turn it into an object has using the passed keys.
	// e.g. --hello world becomes console.log(options.hello); prints "World".

	var opts = {},
		numArgs = args.length;
	for (var i = 0; i < numArgs; i++) {
		if (args[i].indexOf("--") === 0) { // Is this a new key.
			// Yes it is.
			if ((i + 1) < numArgs && args[i + 1].indexOf("--") === 0) { // Is the next arg a key as well?
				// Yes it is, in which case set this key as true.
				opts[args[i].replace("--", "")] = true;
			} else if ((i + 1) < numArgs) { // Ok no it's not.
				// Thus set this key with the next arg as the value
				opts[args[i].replace("--", "")] = args[i + 1];
			}
		}
	}

	this.options = opts;

	// Now let's get a file system handle.
	this.fs = require("fs");
};

QUnitRunner.prototype = {
	verify: function () {
		var ok = false;

		if (typeof this.options.qunit == "undefined") {
			throw new Error("You need to specify where qunit.js lives, like: `--qunit qunit.js`.");
		} else if (!this.fs.isFile(this.options.qunit) || !this.fs.isReadable(this.options.qunit)) {
			throw new Error("Cannot find qunit.js");
		} else {
			ok = true;
		}

		if (typeof this.options.tests == "undefined") {
			throw new Error("You need to specify where your tests live, like: `--tests mytests.js`.");
		} else if (this.options.tests.indexOf(".js") !== -1 && !this.fs.isFile(this.options.tests)) { // Validate file exists if it has .js at the end.
			throw new Error("Test file '"+this.options.tests+"' cannot be found.");
		} else if (!this.fs.isDirectory(this.options.tests)) {
			throw new Error("Cannot find test directory '"+this.options.tests+"'.");
		} else if (!this.fs.isReadable(this.options.tests)) {
			throw new Error("Cannot read test file or directory '"+this.options.tests+"'.");
		} else {
			ok = true; // tests is there and is good to go.
		}

		return ok;
	},
	//
	// Does the actualy running of the tests (either a test file or a folder of tests).
	//
	// @source http://whileonefork.blogspot.com/2011/07/javascript-unit-tests-with-qunit-ant.html
	// @source https://gist.github.com/1363104
	//
	startQunit: function () {
		var self = this,
			testsPassed = 0,
			testsFailed = 0,
			module, moduleStart, testStart, testCases = [],
			current_test_assertions = [],
			junitxml = '<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="testsuites">\n';;

		if (typeof this.options.junit != "undefined") {
			console.log("Going to produce JUnit xml file: "+this.options.junit);
		}

		QUnit.begin({});

		function extend(a, b) {
			for ( var prop in b ) {
				if ( b[prop] === undefined ) {
					delete a[prop];
				} else {
					a[prop] = b[prop];
				}
			}

			return a;
		}

		// Initialize the config, saving the execution queue
		var oldconfig = extend({}, QUnit.config);
		QUnit.init();
		extend(QUnit.config, oldconfig);

		QUnit.testStart = function() {
			testStart = new Date();
		};

		QUnit.moduleStart = function(context) {
			moduleStart = new Date();
			module = context.name;
			testCases = [];
		};

		QUnit.moduleDone = function(context) {
			// context = { name, failed, passed, total }
			var xml = '\t<testsuite name="' + context.name + '" errors="0" failures="' + context.failed + '" tests="' + context.total + '" time="' + (new Date() - moduleStart) / 1000 + '"';
			if (testCases.length) {
				xml += '>\n';
				for (var i = 0, l = testCases.length; i < l; i++) {
					xml += testCases[i];
				}
				xml += '\t</testsuite>';
			} else {
				xml += '/>\n';
			}

			junitxml += xml;
		};

		QUnit.testDone = function(result) {
			if (0 === result.failed) {
				testsPassed++;
			} else {
				testsFailed++;
			}

			console.log((0 === result.failed ? '\033[1;92mPASS\033[0m' : '\033[1;31mFAIL\033[0m') + ' - ' + result.name + ' completed: ');

			// result = { name, failed, passed, total }
			var xml = '\t\t<testcase classname="' + module + '" name="' + result.name + '" time="' + (new Date() - testStart) / 1000 + '"';
			if (result.failed) {
				xml += '>\n';
				for (var i = 0; i < current_test_assertions.length; i++) {
					xml += "\t\t\t" + current_test_assertions[i];
				}
				xml += '\t\t</testcase>\n';
			} else {
				xml += '/>\n';
			}
			current_test_assertions = [];

			testCases.push(xml);
		};

		var running = true;
		QUnit.done = function(i) {
			console.log(testsPassed + ' of ' + (testsPassed + testsFailed) + ' tests successful.');
			console.log('TEST RUN COMPLETED: ' + (0 === testsFailed ? '\033[1;92mWIN\033[0m' : '\033[1;31mFAIL\033[0m'));
			running = false;

			if (typeof self.options.junit != "undefined") {
				junitxml += '</testsuites>';

				// Ok now let's write that xml file to where we told it to. (well actually the user did via the --junit option).
				if (!self.fs.isFile(self.options.junit)) {
					self.fs.write(self.options.junit, junitxml, "w");
				} else {
					console.log("Cannot write junit results file.");
				}
			}
		};

		QUnit.log = function(details) {
			//details = { result , actual, expected, message }
			if (details.result) {
				return;
			}
			var message = details.message || "";
			if (details.expected) {
				if (message) {
					message += ", ";
				}
				message = "expected: " + details.expected + ", but was: " + details.actual;
			}
			var xml = '<failure type="failed" message="' + details.message.replace(/ - \{((.|\n)*)\}/, "") + '"/>\n';

			current_test_assertions.push(xml);
		};

		//Instead of QUnit.start(); just directly exec; the timer stuff seems to invariably screw us up and we don't need it
		QUnit.config.semaphore = 0;
		while( QUnit.config.queue.length ) {
			QUnit.config.queue.shift()();
		}

		// wait for completion
		var ct = 0;
		while ( running ) {
			if (ct++ % 1000000 == 0) {
				console.log('Queue is at ' + QUnit.config.queue.length);
			}

			if (!QUnit.config.queue.length) {
				QUnit.done();
			}
		}

		//exit code is # of failed tests; this facilitates Ant failonerror. Alternately, 1 if testsFailed > 0.
		phantom.exit(testsFailed);
	},
	run: function () {
		//
		// Loads the test file or folder of tests files.
		//
		var loadTests = function () {
			console.log("Load those tests");
			if (this.fs.isFile(this.options.tests)) {
				console.log("Load test file: "+this.options.tests);
				phantom.injectJs(this.options.tests);

				// Now run the tests.
				this.startQunit();
			} else if (this.fs.isDirectory(this.options.tests)) {
				var data = this.fs.list(this.options.tests);

				for (var i = 0; i < data.length; i++) {
					if (data[i].indexOf(".js") !== -1) {
						console.log("Load test file: "+this.options.tests+this.fs.separator+data[i]);
						phantom.injectJs(this.options.tests+this.fs.separator+data[i]);
					}
				}

				// Now run the tests.
				this.startQunit();
			} else {
				throw new Error("Tests is not a file or a directory?! I don't know what to do with that.");
			}
		};

		//
		// Iterates the required scripts detailed in a CommonJS package file, loading them before testing.
		//
		var requirements = function (pkg) {
			if (typeof pkg == "object" && typeof pkg.scripts != "undefined") {
				var parts = this.options.package.split(this.fs.separator);
				parts.pop();
				var path = parts.join(this.fs.separator)+this.fs.separator;

				// Now interate those scripts injecting each one into the current conext.
				for (var key in pkg.scripts) {
					if (pkg.scripts.hasOwnProperty(key)) {
						console.log("Injecting: "+path+pkg.scripts[key]);
						phantom.injectJs(path+pkg.scripts[key]);
					}
				}

				// All scripts loaded, execute the tests.
				loadTests.call(this);
			} else {
				throw new Error("Opsie, package option should be a CommonJS package.json file and should have the option scripts, so we can load those scripts.");
			}
		};

		// First let's verify those options.
		if (this.verify()) {
			console.log("Verified passed options.");

			console.log("Loading QUnit...");
			phantom.injectJs(this.options.qunit);
			console.log("QUnit loaded.");

			if (typeof this.options.package != "undefined") {
				console.log("Found CommonJS package file let's see if we can load it.");

				if (this.fs.isReadable(this.options.package)) {
					console.log("Loading: "+this.options.package);

					var package = "requirements.call(this, "+this.fs.read(this.options.package)+")";

					// Now eval that beast.
					eval(package);
				} else {
					throw new Error("Cannot read package file.");
				}
			}
			else
			{
				console.log("No package file preload, just run those tests...");
				loadTests.call(this);
			}
		}
	}
};

var runner = new QUnitRunner(phantom.args);

runner.run();

I get the following when running the qunit-runner.js script with phantomjs:

TypeError: 'undefined' is not an object

qunit-runner.js:180

Any ideas?

QUnit.log isn't checking "details" is an object. console.log(details) within that function and add a check to verify it's an object at the top of that function.

have updated to include the test, you shouldn't get the above error now. For the record I am not actually using this script anymore, I migrated everything to user busterjs which does all this in a simpler fashion http://busterjs.org/

Should it not be:

if (!details.result || typeof details == "undefined") {

Added the "!"

After adding the "!" it's not throwing that error anymore, but it's alos not creating the xml output file.

It's also producing this error now (for every QUnit test): ReferenceError: Can't find variable: log

Plse ignore the "Can't find variable: log" error.

But, it's showing FAILED for even tests that passed, and is still not creating the output file.

Any ideas?

You might be using a different version of QUnit try this one https://gist.github.com/2488794

Brilliant! Thanks!

The console is now producing the correct results, but the XML output file only contains this:

Have updated the gist, I just checked we still have some tests being run with this on our ci server. Using that version of QUnit and the updated runner file you should be good to go.

I have updated both QUnit.js and the runner with ur latest versions, but still get the same output in the XML file.

try now. I have just run it manually and get xml file correctly with pass and failed tests defined.

sorry by try now, i mean update your file with the file i just updated above and run it.

Nope. Same result:

it wont overwrite the .xml file so make sure u delete it each time.

Yeah, I've tried that, but same results.

can you paste all the output of the command somewhere please (including the command line)

The command that I execute from the command line:
/usr/local/phantomjs/bin/phantomjs qunit-runner.js --qunit /Users/cas/Desktop/POC/baseline/test/qunit/qunit.js --tests /Users/cas/Desktop/POC/baseline/test --junit /Users/cas/Desktop/POC/baseline/test/qunit-results.xml

Command line output:
Verified passed options.
Loading QUnit...
QUnit loaded.
No package file preload, just run those tests...
Load those tests
Load test file: /Users/cas/Desktop/POC/baseline/test/tests.js
Going to produce JUnit xml file: /Users/cas/Desktop/POC/baseline/test/qunit-results.xml
PASS - HTML5 Boilerplate is sweet completed:
Test Environment is good died, exception and test follows
ReferenceError: Can't find variable: log
function () {
expect(3);
ok( !!window.log, "log function present");

var history = log.history && log.history.length || 0;
log("logging from the test suite.")
equals( log.history.length - history, 1, "log history keeps track" )

ok( !!window.Modernizr, "Modernizr global is present");
}
FAIL - Environment is good completed:
Queue is at 0
1 of 2 tests successful.
TEST RUN COMPLETED: FAIL

XML File Content:
"

"

Not too sure then. The only extra info I can give you is all my test files follow the format below.

test("Some description about the test", function () {
ok(true === true, "true is equal to true");
});

XLM Output:

<?xml version="1.0" encoding="UTF-8"?> <testsuites name="testsuites"> </testsuites>

I have created a new test file and included your sample test, but get exactly the same results.

I've added this line junitxml += xml; just after line 157. now the XML produces:
`

`

Running a single test file is not possible.

When providing the location to a JavaScript (.js) file as the value of the argument "--tests", the following exception is thrown:

Error: Cannot find test directory '/Users/stephenmathieson/{...}/phantom-tests.js'.

This is due to your "validate" method (lines 61 through 67). You're throwing an error if the value of "tests" ends in ".js" and is not a file, which is expected. However, if the value of "tests" is an existing JavaScript file, an exception is thrown because it is not a directory.

A simple fix:

    if (this.options.tests === undefined) {
        throw new Error("You need to specify where your tests live, like: `--tests mytests.js`.");
    } else if (this.options.tests.indexOf('.js') !== -1) {
        if (!this.fs.isFile(this.options.tests)) {
            throw new Error("...");
        }
    } else {
        if (!this.fs.isDirectory(...)) {
            throw new Error("...");
        }
    }

I got the latest code. And run it on my local machine. I found that there is bug with the script that the phantomJS will not execute the QUnit.moduleDone event for the last module even it has ready run all the tests of it. I can't figure out how to fix it. Can somebody give me a help please? :)

Am I right in thinking this only finds tests in .js files? It won't find tests in .html files?

As AgileAce says:

I've added this line junitxml += xml; just after line 157.

Needed to get valid (almost) xml.

When i am executing "...../phantomjs-2.1.1-windows\bin>phantomjs.exe qunit-runner.js --qunit ./qunit.js" i am getting error message TypeError: undefined is not an object (evaluating 'args.length') phantomjs://code/qunit-runner.js:30 in QUnitRunner.
I am using Phantomjs 2..1 and all the required files are in same directory i.e. bin of phantomjs.

phantom.args is no longer available in 2.x. You'll need

var system = require('system');
system.args.shift();  // skip first arg (runner)
var args = system.args;

var runner = new QUnitRunner(args);
runner.run()