PhantomJS testing comparison

casper.js

/*global require, console */
(function() {
    "use strict";

    var fs      = require("fs"),
        config  = JSON.parse(fs.read("../../../Tests/CasperJS/data/accounts.json")),
        casper  = require("../../../Tests/CasperJS/casper-factory").create,
        Account = require("../../../Tests/CasperJS/account");

    casper()
        .start()
        .viewport(1440, 800)
        .then(Account.create())
        .thenOpen(config.baseURL + "/login")
        .then(function() {
            this.fill(".login-panel", {
                email    : this.acc.email,
                password : this.acc.password
            }, true);
        })
        .then(function() {
            this.test.assertUrlMatch(config.baseURL + "/account", "User should be redirected to account page on login");
            this.clear();
        })
        .thenOpen(config.baseURL + "/logout")
        .thenOpen(config.baseURL + "/login?redirect_uri=/account/security")
        .then(function() {
            this.fill(".login-panel", {
                email    : this.acc.email,
                password : this.acc.password
            }, true);
        })
        .then(function() {
            this.test.assertUrlMatch(config.baseURL + "/account/security", "Redirects should, well, redirect");
        })
        .thenOpen(config.baseURL + "/logout")
        .then(Account.create("12.12.12.12"))
        .thenOpen(config.baseURL + "/login")
        .then(function() {
            this.fill(".login-panel", {
                email    : this.acc.email,
                password : this.acc.password
            }, true);
        })
        .thenOpen(config.baseURL + "/account/security/email", function() {
            this.click(".auth2f form button");
        })
        .then(function() {
            this.test.assertUrlMatch("done", "Email security enabled");
        })
        .thenOpen(config.baseURL + "/logout")
        .thenOpen(config.baseURL + "/login")
        .then(function() {
            this.fill(".login-panel", {
                email    : this.acc.email,
                password : this.acc.password
            }, true);
        })
        .then(function() {
            if (!this.exists(".login_auth2f_email-page")) {
                this.test.fail("IP Whitelist page not hit");
            } else {
                this.test.pass("WhiteList landing page hit.");
            }
        })
        .run();
}());

dalek.js

/*jshint node:true */

"use strict";

var account = require("./lib/account.js");
    
module.exports = {
    "Redirects to /login" : function(test) {
        test
            .open("http://account.local.ncplatform.net:8888")
            .assert
                .url().is("http://account.local.ncplatform.net:8888/login")
            .done();
    },
    
    "Login form appears" : function(test) {
        test
            .open("http://account.local.ncplatform.net:8888/login")
            .assert.chain()
                .exists("#email")
                .exists("#password")
                .exists("button[type='submit']")
            .end()
            .done();
    },
    
    "Simple login" : function(test) {
        account.create(function(details) {
            test
                .open("http://account.local.ncplatform.net:8888/login")
                .type("#email",    details.email)
                .type("#password", details.password)
                .submit(".login-panel")
                .assert
                    .url().is("http://account.local.ncplatform.net:8888/account/download")
                .open("http://account.local.ncplatform.net:8888/logout")
                .done();
        });
    },
    
    "Login with a redirect" : function(test) {
        account.create(function(details) {
            test
                .open("http://account.local.ncplatform.net:8888/login?redirect_uri=/account/security")
                .type("#email",    details.email)
                .type("#password", details.password)
                .submit(".login-panel")
                .assert
                    .url().is("http://account.local.ncplatform.net:8888/account/security")
                .open("http://account.local.ncplatform.net:8888/logout")
                .done();
            
        });
    },
    
    "IP Whitelisting" : function(test) {
        account.create("12.12.12.12", function(details) {
            test
                .open("http://account.local.ncplatform.net:8888/login?redirect_uri=/account/security/email")
                .type("#email",    details.email)
                .type("#password", details.password)
                .submit(".login-panel")
                .submit(".auth2f.email > form")
                .assert
                    .url().is("http://account.local.ncplatform.net:8888/account/security/email/done")
                .open("http://account.local.ncplatform.net:8888/logout")
                .open("http://account.local.ncplatform.net:8888/login")
                .type("#email",    details.email)
                .type("#password", details.password)
                .submit(".login-panel")
                .assert.chain()
                    .url().is("http://account.local.ncplatform.net:8888/login/wait?type=email&redirect_uri=%2Faccount%2Fdownload")
                    .exists(".login_auth2f_email-page")
                .end()
                .done();
        });
    }
};

thoughts.md

Thoughts

So far I really like the more-organized nature of the DalekJS tests. I also really strongly prefer writing tests in NodeJS vs. PhantomJS' looks-like-node-but-isn't environment.

I haven't run perf tests yet, but I suspect CasperJS will be a bit faster since it's slightly more to-the-metal. It does mean that our Casper tests cannot be run against other browsers like DalekJS but for my needs that isn't hugely important.

One huge point against DalekJS is that so far I haven't been able to determine a way to get content out of the PhantomJS instance. We have tests to add TOTP auth to an account that are currently just impossible to run unless we provide better backside hooks. Being able to read from the remote DOM would be hugely useful, even if it's more limited than Casper's ability to evaluate code within the page context.

I also fought with DalekJS for a bit over dalekjs/dalek#1, which combined with the 0.0.1 version should give anybody thinking of whole-heartedly switching at least a bit of pause.