Plan to implement a Selenium grid for testing AlloyUI

ci.js

var webdriver = require('../webdriver.json'),
    async = require('async'),
    gulp = require('gulp'),
    path = require('path'),
    CWD = path.join(process.env.INIT_CWD, 'src'),
    spawn = require('spawn-local-bin'),
    run = require('run-sequence');

function escape(string) {
    return string.replace(/([;=])/g, '\\$1');
}

function getCapabilities(raw) {
    var prop,
        properties = '';

    for (prop in raw) {
        if (raw.hasOwnProperty(prop)) {
            properties += escape(prop) + '=' + escape(raw[prop]) + ';';
        }
    }

    return properties;
}

gulp.task('ci', function (callback) {
    async.mapSeries(webdriver.capabilities, function (capabilities, res) {
        var cmd = 'yeti',
            caps = getCapabilities(capabilities),
            args = ['--wd-url', webdriver.host, '--caps', caps];

        spawn(cmd, args, CWD)
            .on('exit', function(code) {
                if (code !== 0) {
                    console.error('Test failed with \'' + caps);
                }

                res(null, code);
            });
    }, function (err, result) {
        var success = result.filter(function (each) {
            return each !== 0;
        }).length === 0;

        if (!success) {
            console.error('Some tests failed.');
            process.nextTick(function () {
                process.exit(1);
            });
        }

        callback();
    });
});

example.log

$ gulp ci
[15:59:00] Using gulpfile ~/projects/liferay/alloy-ui/gulpfile.js
[15:59:00] Starting 'ci'...
Found 74 files to test.
Creating a Hub. Open `yeti --server` in another Terminal to reuse browsers for future batches.

Waiting for Hub to launch browsers...
  Agent connected: Firefox (30.0) / Mac OS from ::ffff:192.168.70.1
✓ Testing started on Firefox (30.0) / Mac OS
✗ Script error: uncaught exception: ComparisonFailure: Values should be equal.
Expected: # (string)
Actual: #mdo (string)
  URL: src/aui-scrollspy/tests/unit/index.html
  Line: 0
  User-Agent: Firefox (30.0) / Mac OS
✗ aui-tooltip on Firefox (30.0) / Mac OS
   in Tooltips
     should update position on scrolling from .tooltip on triggerScroll: Trigger are out of viewport and there is a scroll on page, so the tooltip should be moved as well
       Expected: 1206.5333251953125 (number)
       Actual: 1186.5333251953125 (number)

✓ Agent completed: Firefox (30.0) / Mac OS
✗ Failures: 1 of 371 tests failed. (1 minutes, 2 seconds)
  Agent disconnected: Firefox (30.0) / Mac OS from ::ffff:192.168.70.1
Test failed with 'browserName=firefox;
Found 74 files to test.
Creating a Hub. Open `yeti --server` in another Terminal to reuse browsers for future batches.

Waiting for Hub to launch browsers...
  Agent connected: Chrome (36.0.1985.125) / Mac OS from ::ffff:192.168.70.1
✓ Testing started on Chrome (36.0.1985.125) / Mac OS
✓ Agent completed: Chrome (36.0.1985.125) / Mac OS
✓ 371 tests passed! (54 seconds)
  Agent disconnected: Chrome (36.0.1985.125) / Mac OS from ::ffff:192.168.70.1
Some tests failed.
[16:00:58] Finished 'ci' after 1.95 min

hub.json

{
	"timeout": 60000,
	"browserTimeout": 60000
}

ie10.json

{
    "capabilities": [
        {
            "browserName": "internet explorer",
            "version": "10",
            "platform": "VISTA",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        }
    ]   ,
    "configuration": {
    }
}

ie11.json

{
    "capabilities": [
        {
            "browserName": "internet explorer",
            "version": "11",
            "platform": "VISTA",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        }
    ]   ,
    "configuration": {
    }
}

ie8.json

{
    "capabilities": [
        {
            "browserName": "internet explorer",
            "version": "8",
            "platform": "VISTA",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        }
    ]   ,
    "configuration": {
    }
}

ie9.json

{
    "capabilities": [
        {
            "browserName": "internet explorer",
            "version": "9",
            "platform": "VISTA",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        }
    ]   ,
    "configuration": {
    }
}

mac.json

{
    "capabilities": [
        {
            "browserName": "firefox",
            "version": "30",
            "firefox_binary": "/Applications/Firefox.app/Contents/MacOS/firefox-bin",
            "platform": "MAC",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        },
        {
            "browserName": "chrome",
            "version": "36",
            "chrome_binary": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
            "platform": "MAC",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        },
        {
            "browserName": "safari",
            "version": "7",
            "chrome_binary": "/Applications/Safari.app/Contents/MacOS/Safari",
            "platform": "MAC",
            "maxInstances": 1,
            "seleniumProtocol": "WebDriver"
        }
    ]
}

SeleniumGrid.md

AlloyUI Selenium Grid Config

Selenium grid configuration for the functional testing of AlloyUI.

Running all the tests on all the environment takes some time. On average it takes 1min to run all the tests on Chrome on a last-generation MacBook Pro. Running on all the browsers we plan to support should take about 1 hour.

Dependencies

• Selenium standalone server
• Selendroid
• ios-driver
• Android Studio
• iOS Simulator

Test environments (capabilities)

It is important to test the AlloyUI framework with major browsers. Both desktop and mobile.

The following environments were chosen for this first step using the Selenium grid.

• Google Chrome (Linux or OS X)
• Firefox (Linux or OS X)
• Safari (OS X)
• Safari mobile (simulator on OS X)
• Android (simulator on Linux or OS X)
• Internet Explorer 8, 9, 10, 11 (Windows)

Unless specified, we use the latest available version for operating systems, web drivers, and browsers.

Free virtual machines with different versions of Internet Explorer can be downloaded from modern.IE.

CI task

Run gulp ci to run the integration tests.

The tests suites are run in series (one browser after the other) with yeti reading the capabilities.json file.

We can make the task faster making the tests run in parallel in the future.

If we decide to do this the following should be considered:

• yeti's integration with Selenium needs improvement to run tests in parallel without waiting for all browsers to be available (what wastes Selenium availability)
• we need to keep track of each browser log individually without getting a mengled log
• focus issue (more on that later)

Hub

The hub works like a bridge between the Selenium clients (test runners) and the nodes (browser drivers).

Start the HUB with

java -jar selenium-server-standalone-<version>.jar -role hub -hubConfig hub.json

Nodes

Start a node with

java -jar selenium-server-standalone-<version>.jar -role node -hub <hub> -nodeConfig node.json

Where hub is the hub HTTP address.

To pass a system property use -DpropertyName=value.

Browser drivers

The Selenium standalone server already comes with some browser drivers, but you may need to install another driver. For example, to use Internet Explorer you must get a driver fot it.

Download the InternetExplorerDriver.

To execute it, you use something like java -jar selenium-server-standalone-<version>.jar -role node -hub http://192.168.70.1:4444/ -Dwebdriver.ie.driver="IEDriverServer.exe" -nodeConfig ie8.json

Some issues

Focus

Some tests might fail if the browser isn't in focus. Also, commands sent to IE might not work.

This is why it might be a good idea to have a single browser running on a Selenium node.

Internet Explorer 11 support

Verify the issue IE11 exceptions with IEDriverServer: Unable to get browser and edit the Windows Registry accordingly to make the IE11 driver work.

Others

• Browser zoom level not set to 100% made Selenium throw an exception on IE.
• If yeti dies for some unknown reason a browser may hang for a few minutes before being killed by Selenium

Reference: https://code.google.com/p/selenium/wiki/InternetExplorerDriver#Browser_Focus

Testing on Android (Galaxy Nexus)

Install Android Studio and create an Android System Image with a Nexus virtual device running the latest Android version inside it. It'll create an ~/.android directory with the virtual machine.

Then, run a Selendroid node and register it on the HUB.

Testing on iOS

Make sure you have Xcode and the iOS Simulator installed on your system.

Then download the ios-driver. See the basics of how to patch your iPhoneSimulator to work with it, and add it as a node on the grid. It's similar to Selendroid.

Docs

• Distributed Testing with Selenium Grid
• Selenium Documentation

webdriver.json

{
    "host": "http://localhost:4444/",
    "capabilities": [
        {
            "browserName": "chrome"
        },
        {
           "browserName": "firefox"
        },
        {
            "browserName": "safari",
            "platform": "MAC"
        },
        {
            "browserName": "safari",
            "platform": "iOS"
        },
        {
            "browserName": "android"
        },
        {
            "browserName": "internet explorer",
            "version": "8"
        },
        {
            "browserName": "internet explorer",
            "version": "9"
        },
        {
            "browserName": "internet explorer",
            "version": "10"
        },
        {
            "browserName": "internet explorer",
            "version": "11"
        }
    ]
}