A Testem issue

5-a-testem-issue.md

The failed tests are not been reported on testem as we expected.

hello.js

function hello(name){
    return "hello___ " + (name || 'world');
}

hello_spec.js

describe('hello', function(){
    it('should say hello', function(){
        expect(hello()).toBe('hello world');
    });
    it('should say hello to person', function(){
        expect(hello('Bob')).toBe('hello Bob');
    });
});

package.json

{
  "name": "saucelabs",
  "description": "example",
  "version": "0.0.1",
  "devDependencies": {
    "sauce-connect-launcher": "~0.1.11",
    "optimist":               "~0.6.0",
    "wd":                     "~0.0.34",
    "async":                  "~0.2.9",
    "q":                      "~0.9.6",
    "request":                "~2.25.0"
  }
}

sauce-config.js

var webdriver   = require('wd'),
    async       = require('async'),
    argv        = require('optimist').argv,
    exports     = module.exports = {},
    libraryName = exports.libraryName = "Testem",
    auth        = exports.auth = {
      username:  process.env.SAUCE_USERNAME,
      accessKey: process.env.SAUCE_ACCESS_KEY,
      build:     process.env.TRAVIS_BUILD_NUMBER || "dev-tests",
      tunnelIdentifier: process.env.TRAVIS_JOB_NUMBER || libraryName
    },

    host = "ondemand.saucelabs.com",
    port = 80,
    browser = exports.browser = webdriver.remote(host, port, auth.username, auth.accessKey),

    desired = exports.desired = {
      "browserName": argv.browserNameSL || "chrome",
      "version"    : argv.versionSL     || "",
      "platform"   : argv.platformSL    || "Linux",
      "tags"       : [libraryName, "test"],
      "name"       : libraryName + " tests",
      "public"     : "public",
      "build"      : auth.build,
      "tunnel-identifier": auth.tunnelIdentifier,
      "record-video": true
    },

    launcherOptions = exports.launcherOptions = {
      username: auth.username,
      accessKey: auth.accessKey,
      verbose: true,
      logfile: 'sauce-example.log', //optionally change sauce connect logfile location
      tunnelIdentifier: auth.tunnelIdentifier, // optionally identity the tunnel for concurrent tunnels
      logger: console.log,
      no_progress: false // optionally hide progress bar
    };


browser.on("status", function(info) {
  //console.log("\x1b[36m%s\x1b[0m", info);
});

browser.on("command", function(meth, path, data) {
  //console.log(" > \x1b[33m%s\x1b[0m: %s", meth, path, data || "");
});

process.on('uncaughtException', function(err) {
  browser.quit();
  console.error('Caught exception: ' + err.stack );
});

sauce-javascript-tests-integration.js

/**
 * JavaScript tests integration with Sauce
 * https://saucelabs.com/docs/javascript-unit-tests-integration
 */
var config  = require('./sauce-config.js'),
    updateJobStatus = require('./sauce-update-job-status.js'),
    async   = require('async'),
    browser = config.browser,
    desired = config.desired;

module.exports = function(localhost_url, script_for_sauce_data_schemas, callback) {
  async.waterfall([
    function(callback) {
      browser.init(desired, function(err){
        callback(err);
      });
    },

    function(callback) {
      browser.get(localhost_url, function(err){
        callback(err);
      });
    },

    function(callback) {
      setTimeout( function() {
        callback( null );
      }, 5000 );
    },

    function(callback) {
      updateJobStatus(script_for_sauce_data_schemas, callback);
    },

    function(result, callback) {
      browser.quit(function(err){
        callback(err);
      });
    }

  ], function(err) {
    err && console.error('Caught exception: ' + err.stack);
    callback && callback(err);
  });
};

sauce-launcher-webdriver.js

var launcher = require('sauce-connect-launcher'),
    config   = require('./sauce-config.js'),
    integrationTest = require('./sauce-javascript-tests-integration.js');

launcher(config.launcherOptions, function (err, sauceConnectProcess) {
  console.log("Started Sauce Connect Process");

  integrationTest(
    "http://localhost:8080",
    "{failedCount: jasmine.currentEnv_.currentRunner_.results().failedCount}",

    function() {
      sauceConnectProcess.close(function () {
        console.log("Closed Sauce Connect process");
      });
    }

  );
});

sauce-rest-api-update-job-status.js

var Q       = require("q"),
    request = require("request"),
    config  = require('./sauce-config.js'),
    browser = config.browser,
    auth    = config.auth;

module.exports = function (data) {
  var deferred = Q.defer(),
      url  = ["/v1/", auth.username, "/jobs/", browser.sessionID].join("");

  request({
    method: "PUT",
    uri: ["https://", auth.username, ":", auth.accessKey, "@saucelabs.com/rest", url].join(""),
    headers: {"Content-Type": "application/json"},
    body: JSON.stringify(data)
  }, function (error, response, body) {
    deferred.resolve(response.body);
  });
  return deferred.promise;
};

sauce-update-job-status.js

var async   = require('async'),
    config  = require('./sauce-config.js'),
    api     = require('./sauce-rest-api-update-job-status.js'),
    browser = config.browser,
    auth    = config.auth,

    waitUntilResultsAreAvailable = function(js_script, timeout, start, callback) {
      var now = new Date();
      start = start || now;

      if (now - start > timeout) {
        callback( new Error("Timeout: Element not there") );
      } else {
        browser.eval(js_script, function(err, jsValue) {
          if (jsValue !== null) callback(null, {resultScript: jsValue});
          else waitUntilResultsAreAvailable(js_script, timeout, start, callback);
        });
      }
    };

module.exports = function(script_for_sauce_data_schemas, callback) {
  async.waterfall([

    function(callback) {
      waitUntilResultsAreAvailable(script_for_sauce_data_schemas, 15000, null, callback);
    },

    function(obj, callback) {
      setTimeout( function() {
        callback( null, obj );
      }, 5000 );
    },

    function(obj, callback) {
      var data = resultScript = obj.resultScript || {};
      data.passed = resultScript.passed || resultScript.failedCount === 0;

      api(data).then( function(body) {
        obj.body = body;
        console.warn("Check out test results at http://saucelabs.com/jobs/" + browser.sessionID + "\n");
        callback(null, obj);
      });
    }
  ], function(err, result) {
    callback(err, result);
  });
};

testem-ci-port-8080.txt

_

testem-ci.txt

_

testem.yaml

framework: jasmine

on_start:
    command: npm install

src_files:
    - hello.js
    - hello_spec.js

launchers:
    SL_iPad_6:
        command: node sauce-launcher-webdriver.js --browserNameSL='ipad' --versionSL='6' --platformSL='OS X 10.8'

    SL_Chrome:
        command: node sauce-launcher-webdriver.js --browserNameSL='chrome' --versionSL='' --platformSL='Linux'

    SL_IE_7:
        command: node sauce-launcher-webdriver.js --browserNameSL='internet explorer' --versionSL='7' --platformSL='Windows XP'

launch_in_ci: [SL_iPad_6]
launch_in_dev: [phantomjs]