Repro geckodriver performance bug

README.md

repro-geckodriver-perf-bug

Node.js project to repro the bug that seems to cause geckodriver to report longer page load times than from manual testing. It's not clear whether these load times are truly slower or if it's just the reported values that are higher.

The library used is selenium-standalone, which I've confirmed is actually downloading geckodriver 0.9.0 (and this can be found in node_modules/selenium-standalone/.selenium/). The project is shrinkwrapped to ensure that the same exact version of all dependencies are used.

To run the test, just do:

npm install
npm start

Then Firefox will be launched, the tests will be run, and the performance metrics will be written to stdout.

index.js

var selenium = require('selenium-standalone')
var wd = require('wd')
var denodeify = require('denodeify')
var times = require('lodash.times')

var installSelenium = denodeify(selenium.install.bind(selenium))
var startSelenium = denodeify(selenium.start.bind(selenium))

var SELENIUM_VERSION = '2.53.1'

var seleniumPromise = Promise.resolve().then(() => {
  var seleniumOpts = { version: SELENIUM_VERSION }
  return installSelenium(seleniumOpts).then(() => {
    return startSelenium(seleniumOpts)
  })
})

function getLoadData (browser) {
  return new Promise(resolve => setTimeout(resolve, 2000)).then(() => {
    var script = '[' +
      'performance.timing.navigationStart, ' +
      'performance.timing.loadEventEnd, ' +
      'performance.timing.domContentLoadedEventEnd' +
      ']'
    return browser.eval(script)
  }).then(res => {
    if (res.filter(x => x === 0).length) { // one of the timing events is 0
      // retry
      return getLoadData(browser)
    }

    return {
      load: res[1] - res[0],
      DOMContentLoaded: res[2] - res[0]
    }
  })
}

function loadPage (opts) {
  return Promise.resolve().then(() => {
    return seleniumPromise.then(() => {
      var client = wd.promiseChainRemote()
      var tunnelId = Math.random()
      var clientOpts = {
        browserName: opts.browser,
        tunnelTimeout: 30 * 60 * 1000,
        name: client.browser + ' - ' + tunnelId,
        'max-duration': 60 * 45,
        'command-timeout': 599,
        'idle-timeout': 599,
        'tunnel-identifier': tunnelId
      }
      var browser = client.init(clientOpts)
      var results = []
      var promise = Promise.resolve()
        .then(() => browser.maximize())
        .then(() => browser.get('about:blank'))
        .then(() => new Promise(resolve => setTimeout(resolve, 2000)))
      times(opts.iterations, i => {
        promise = promise.then(() => {
          return browser.get(opts.url)
        }).then(() => {
          return getLoadData(browser)
        }).then(loadData => {
          loadData.iteration = i
          results.push(loadData)
        })
      })
      return promise.then(() => {
        return client.quit()
      }).then(() => {
        return results
      })
    })
  })
}

function series (promiseFactories) {
  var promise = Promise.resolve()
  promiseFactories.forEach(factory => {
    promise = promise.then(factory)
  })
  return promise
}

var urls = [
  'http://mozilla.com',
  'http://google.com',
  'http://bing.com'
]
series(urls.map(url => {
  var opts = {
    browser: 'firefox',
    url: url,
    iterations: 10
  }
  return function () {
    console.log('testing...', opts)
    return loadPage(opts).then(results => {
      console.log(JSON.stringify(results, null, '  '))
    })
  }
})).then(() => {
  console.log('done!')
  process.exit(0)
}).catch(err => {
  console.log(err)
  console.log(err.stack)
  process.exit(1)
})

npm-shrinkwrap.json

{
  "name": "repro-geckodriver-perf-bug",
  "version": "0.1.0",
  "npm-shrinkwrap-version": "200.5.1",
  "node-version": "v6.2.2",
  "dependencies": {
    "ansi-regex": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
    },
    "ansi-styles": {
      "version": "2.2.1",
      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz"
    },
    "archiver": {
      "version": "0.14.4",
      "resolved": "https://registry.npmjs.org/archiver/-/archiver-0.14.4.tgz",
      "dependencies": {
        "async": {
          "version": "0.9.2",
          "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
        },
        "lodash": {
          "version": "3.2.0",
          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz"
        }
      }
    },
    "asn1": {
      "version": "0.1.11",
      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz"
    },
    "assert-plus": {
      "version": "0.1.5",
      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz"
    },
    "async": {
      "version": "1.2.1",
      "resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz"
    },
    "aws-sign2": {
      "version": "0.5.0",
      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz"
    },
    "balanced-match": {
      "version": "0.4.1",
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.1.tgz"
    },
    "bl": {
      "version": "0.9.5",
      "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz"
    },
    "bluebird": {
      "version": "2.10.2",
      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.10.2.tgz"
    },
    "boom": {
      "version": "0.4.2",
      "resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz"
    },
    "brace-expansion": {
      "version": "1.1.5",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.5.tgz"
    },
    "buffer-crc32": {
      "version": "0.2.5",
      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.5.tgz"
    },
    "caseless": {
      "version": "0.8.0",
      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.8.0.tgz"
    },
    "chalk": {
      "version": "1.1.3",
      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz"
    },
    "combined-stream": {
      "version": "0.0.7",
      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz"
    },
    "commander": {
      "version": "2.6.0",
      "resolved": "https://registry.npmjs.org/commander/-/commander-2.6.0.tgz"
    },
    "compress-commons": {
      "version": "0.2.9",
      "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-0.2.9.tgz"
    },
    "concat-map": {
      "version": "0.0.1",
      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
    },
    "core-util-is": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
    },
    "crc32-stream": {
      "version": "0.3.4",
      "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-0.3.4.tgz"
    },
    "cryptiles": {
      "version": "0.2.2",
      "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz"
    },
    "ctype": {
      "version": "0.5.3",
      "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz"
    },
    "delayed-stream": {
      "version": "0.0.5",
      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
    },
    "denodeify": {
      "version": "1.2.1",
      "resolved": "https://registry.npmjs.org/denodeify/-/denodeify-1.2.1.tgz"
    },
    "end-of-stream": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.1.0.tgz"
    },
    "escape-string-regexp": {
      "version": "1.0.5",
      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
    },
    "fd-slicer": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz"
    },
    "forever-agent": {
      "version": "0.5.2",
      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.5.2.tgz"
    },
    "form-data": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.2.0.tgz",
      "dependencies": {
        "async": {
          "version": "0.9.2",
          "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
        },
        "mime-types": {
          "version": "2.0.14",
          "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.14.tgz"
        }
      }
    },
    "generate-function": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz"
    },
    "generate-object-property": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz"
    },
    "glob": {
      "version": "4.3.5",
      "resolved": "https://registry.npmjs.org/glob/-/glob-4.3.5.tgz"
    },
    "graceful-readlink": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz"
    },
    "har-validator": {
      "version": "1.8.0",
      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz",
      "dependencies": {
        "commander": {
          "version": "2.9.0",
          "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz"
        }
      }
    },
    "has-ansi": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz"
    },
    "hawk": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/hawk/-/hawk-1.1.1.tgz"
    },
    "hoek": {
      "version": "0.9.1",
      "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz"
    },
    "http-signature": {
      "version": "0.10.1",
      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.1.tgz"
    },
    "inflight": {
      "version": "1.0.5",
      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.5.tgz"
    },
    "inherits": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
    },
    "is-absolute": {
      "version": "0.1.7",
      "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-0.1.7.tgz"
    },
    "is-my-json-valid": {
      "version": "2.13.1",
      "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz"
    },
    "is-property": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz"
    },
    "is-relative": {
      "version": "0.1.3",
      "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-0.1.3.tgz"
    },
    "isarray": {
      "version": "0.0.1",
      "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
    },
    "isstream": {
      "version": "0.1.2",
      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
    },
    "json-stringify-safe": {
      "version": "5.0.1",
      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
    },
    "jsonpointer": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz"
    },
    "lazystream": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-0.1.0.tgz"
    },
    "lodash": {
      "version": "3.9.3",
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.9.3.tgz"
    },
    "lodash._baseiteratee": {
      "version": "4.7.0",
      "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz"
    },
    "lodash._basetostring": {
      "version": "4.12.0",
      "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz"
    },
    "lodash._stringtopath": {
      "version": "4.8.0",
      "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz"
    },
    "lodash.times": {
      "version": "4.2.0",
      "resolved": "https://registry.npmjs.org/lodash.times/-/lodash.times-4.2.0.tgz"
    },
    "mime-db": {
      "version": "1.12.0",
      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.12.0.tgz"
    },
    "mime-types": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-1.0.2.tgz"
    },
    "minimatch": {
      "version": "2.0.10",
      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-2.0.10.tgz"
    },
    "minimist": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.1.0.tgz"
    },
    "mkdirp": {
      "version": "0.5.0",
      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz",
      "dependencies": {
        "minimist": {
          "version": "0.0.8",
          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz"
        }
      }
    },
    "node-int64": {
      "version": "0.3.3",
      "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz"
    },
    "node-uuid": {
      "version": "1.4.7",
      "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz"
    },
    "oauth-sign": {
      "version": "0.5.0",
      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.5.0.tgz"
    },
    "once": {
      "version": "1.3.3",
      "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz"
    },
    "pend": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz"
    },
    "progress": {
      "version": "1.1.8",
      "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz"
    },
    "q": {
      "version": "1.4.1",
      "resolved": "https://registry.npmjs.org/q/-/q-1.4.1.tgz"
    },
    "qs": {
      "version": "2.3.3",
      "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz"
    },
    "readable-stream": {
      "version": "1.0.34",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz"
    },
    "request": {
      "version": "2.51.0",
      "resolved": "https://registry.npmjs.org/request/-/request-2.51.0.tgz"
    },
    "selenium-standalone": {
      "version": "5.5.0",
      "resolved": "https://registry.npmjs.org/selenium-standalone/-/selenium-standalone-5.5.0.tgz"
    },
    "sntp": {
      "version": "0.2.4",
      "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz"
    },
    "string_decoder": {
      "version": "0.10.31",
      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
    },
    "stringstream": {
      "version": "0.0.5",
      "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
    },
    "strip-ansi": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz"
    },
    "supports-color": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz"
    },
    "tar-stream": {
      "version": "1.1.5",
      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz"
    },
    "tough-cookie": {
      "version": "2.2.2",
      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz"
    },
    "tunnel-agent": {
      "version": "0.4.3",
      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
    },
    "underscore.string": {
      "version": "3.0.3",
      "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.0.3.tgz"
    },
    "urijs": {
      "version": "1.16.1",
      "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.16.1.tgz"
    },
    "vargs": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/vargs/-/vargs-0.1.0.tgz"
    },
    "wd": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/wd/-/wd-0.4.0.tgz",
      "dependencies": {
        "async": {
          "version": "1.0.0",
          "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz"
        },
        "boom": {
          "version": "2.10.1",
          "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
        },
        "caseless": {
          "version": "0.9.0",
          "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.9.0.tgz"
        },
        "cryptiles": {
          "version": "2.0.5",
          "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
        },
        "forever-agent": {
          "version": "0.6.1",
          "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
        },
        "hawk": {
          "version": "2.3.1",
          "resolved": "https://registry.npmjs.org/hawk/-/hawk-2.3.1.tgz"
        },
        "hoek": {
          "version": "2.16.3",
          "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
        },
        "mime-types": {
          "version": "2.0.14",
          "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.14.tgz"
        },
        "oauth-sign": {
          "version": "0.6.0",
          "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.6.0.tgz"
        },
        "qs": {
          "version": "2.4.2",
          "resolved": "https://registry.npmjs.org/qs/-/qs-2.4.2.tgz"
        },
        "request": {
          "version": "2.55.0",
          "resolved": "https://registry.npmjs.org/request/-/request-2.55.0.tgz"
        },
        "sntp": {
          "version": "1.0.9",
          "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
        }
      }
    },
    "which": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/which/-/which-1.1.1.tgz"
    },
    "wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
    },
    "xtend": {
      "version": "4.0.1",
      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz"
    },
    "yauzl": {
      "version": "2.6.0",
      "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.6.0.tgz"
    },
    "zip-stream": {
      "version": "0.5.2",
      "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-0.5.2.tgz",
      "dependencies": {
        "lodash": {
          "version": "3.2.0",
          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz"
        }
      }
    }
  }
}

package.json

{
  "name": "repro-geckodriver-perf-bug",
  "version": "0.1.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "start": "node index.js"
  },
  "author": "Nolan Lawson <nolan@nolanlawson.com>",
  "license": "MIT",
  "private": true,
  "dependencies": {
    "denodeify": "1.2.1",
    "lodash.times": "4.2.0",
    "selenium-standalone": "5.5.0",
    "wd": "0.4.0"
  }
}