Simple website load tester written in Node.js

loadtest.js

/*jslint node: true, white: true, nomen: true, plusplus: true*/
/*globals require: true, process: true*/

/*!
 * Generic load tester
 * Copyright (c) 2013 Leftclick.com.au
 * MIT License, or whatever.
 * Author assumes no liability for damage, loss, etc.
 *
 * Configuration is passed as a command-line argument when running node, e.g.
 *
 * node loadtest.js ./data-example.json
 *
 * The format of this JSON file is as per the "defaults" variable explained below.
 */
(function (_, request, config) {
	"use strict";

	var log, urlToAbsolute, queueRequest, makeRequest, handleOkResponse, handleRedirectResponse, handleErrorResponse,
		getLinkedContentUrls, requestLinkedContent, startWorkflow,
		testStarted, workflowIndex, requestIndex, lastRequest, cache = {},
		defaults = {
			linkedContentUrlPatterns: [
				'^[\\s\\S]*?link.*?href="(.*?\\.css)"([\\s\\S]*)$',
				'^[\\s\\S]*?img.*?src="(.*?)"([\\s\\S]*)$',
				'^[\\s\\S]*?script.*?src="(.*?)"([\\s\\S]*)$'
			],
			tuning: {
				maximumRuntime: 300000,
				pageRequestInterval: 1,
				clearCachePerWorkflow: true
			},
			logging: {
				http: true,
				html: false,
				form: true,
				exit: true,
				error: '### ERROR: {message}',
				redirect: 'Redirecting to: {message}'
			}
		};

	// Log a message of the given type.
	log = function (type, message) {
		if (config.logging[type]) {
			if (typeof config.logging[type] === 'string') {
				message = config.logging[type].replace('{message}', message);
			}
			console.log(message);
		}
	};

	// Convert a relative or absolute URL to an absolute URL, with the given reference URL to supply missing domain.
	urlToAbsolute = function (url, referenceUrl) {
		return url.match(/^https?:\/\//) ? url : referenceUrl.replace(/^(https?:\/\/[a-zA-Z0-9\-_.]*?)\/[a-zA-Z0-9\-_.\?&%~\/]*$/, '$1') + url;
	};

	// Queue the next request.
	queueRequest = function () {
		setTimeout(function () {
			if (Date.now() < testStarted + config.tuning.maximumRuntime) {
				makeRequest(config.workflows[workflowIndex].requests[requestIndex]);
			} else {
				log('exit', 'Maximum runtime exceeded');
				process.exit();
			}
		}, config.tuning.pageRequestInterval);
	};

	// Send the request, and handle the response.
	makeRequest = function (requestOptions) {
		var requestSent = Date.now();
		requestOptions.jar = true;
		requestOptions.form = requestOptions.form || {};
		if (lastRequest) {
			_.each(requestOptions.form, function (field, key) {
				if (typeof field === 'object') {
					requestOptions.form[key] = lastRequest.responseText.replace(new RegExp(field.pattern), field.replacement);
				}
			});
		}
		if (requestOptions.hasOwnProperty('url') && typeof requestOptions.url === 'object') {
			requestOptions.url = urlToAbsolute(lastRequest.responseText.replace(new RegExp(requestOptions.url.pattern), requestOptions.url.replacement), lastRequest.options.url);
		}
		if (!requestOptions.hasOwnProperty('url') || !requestOptions.url.match(/^https?:\/\//)) {
			log('error', 'Bad URL string of length ' + requestOptions.url.length);
			startWorkflow();
		} else {
			log('http', 'Making request ' + requestIndex + ' in workflow ' + workflowIndex + ': ' + config.workflows[workflowIndex].label);
			log('http', requestOptions.method.toUpperCase() + ' ' + requestOptions.url);
			log('form', requestOptions.form);
			request(requestOptions, function (error, httpResponse, responseText) {
				log('http', 'Received ' + (!responseText ? 'empty response' : (error ? 'error ' : '') + 'response of length ' + responseText.length) + ' with status code: ' + (httpResponse && httpResponse.hasOwnProperty('statusCode') ? httpResponse.statusCode : '[unknown]') + ' in ' + (Date.now() - requestSent) + ' ms');
				log('html', responseText);
				if (!error && httpResponse.statusCode.toString().substring(0, 1) === '2') {
					handleOkResponse(requestOptions, responseText);
				} else if (!error && httpResponse.statusCode.toString().substring(0, 1) === '3') {
					handleRedirectResponse(requestOptions, httpResponse);
				} else {
					handleErrorResponse(httpResponse);
				}
			});
		}
	};

	// Handle an "OK" response.
	handleOkResponse = function (requestOptions, responseText) {
		requestLinkedContent(getLinkedContentUrls(responseText, requestOptions.url), function () {
			lastRequest = {
				options: requestOptions,
				responseText: responseText
			};
			if (++requestIndex >= config.workflows[workflowIndex].requests.length) {
				startWorkflow();
			} else {
				queueRequest();
			}
		});
	};

	// Handle a "Redirect" response.
	handleRedirectResponse = function (requestOptions, httpResponse) {
		if (!httpResponse.headers.hasOwnProperty('location') || !httpResponse.headers.location) {
			log('error', 'Redirect response received with no location header');
			startWorkflow();
		} else if (!requestOptions.hasOwnProperty('expectedRedirectUrlPattern') || !httpResponse.headers.location.match(new RegExp(requestOptions.expectedRedirectUrlPattern))) {
			log('error', 'Received unexpected redirect: ' + httpResponse.headers.location);
			startWorkflow();
		} else {
			log('redirect', httpResponse.headers.location);
			makeRequest({
				method: 'GET',
				url: urlToAbsolute(httpResponse.headers.location, requestOptions.url)
			});
		}
	};

	// Handle an "Error" response.
	handleErrorResponse = function (httpResponse) {
		log('error', 'Unhandled or error response code received: ' + (httpResponse && httpResponse.hasOwnProperty('statusCode') ? httpResponse.statusCode : '[unknown]'));
		startWorkflow();
	};

	// Get the list of "linked content" URLs presented in the given response text.
	getLinkedContentUrls = function (responseText, referenceUrl) {
		var regex, responseTextCopy,
			linkedContentUrls = [];
		_.each(config.linkedContentUrlPatterns, function (pattern) {
			regex = new RegExp(pattern);
			responseTextCopy = responseText;
			while (responseTextCopy.match(regex)) {
				linkedContentUrls.push(urlToAbsolute(responseTextCopy.replace(regex, '$1'), referenceUrl));
				responseTextCopy = responseTextCopy.replace(regex, '$2');
			}
		});
		return linkedContentUrls;
	};

	// Request the next "linked content" URL, and when it is loaded, request the next one; when all are loaded, invoke
	// the callback given by done.
	requestLinkedContent = function (linkedContentUrls, done) {
		var linkedContentUrl, requestSent = Date.now();
		if (linkedContentUrls.length > 0) {
			linkedContentUrl = linkedContentUrls[0];
			if (!cache.hasOwnProperty(linkedContentUrl)) {
				log('http', 'Loading linked content: ' + linkedContentUrl);
				request(linkedContentUrl, function (error, httpResponse, responseText) {
					log('http', 'Received ' + (!responseText ? 'empty response' : (error ? 'error ' : '') + 'response of length ' + responseText.length) + ' with status code: ' + (httpResponse && httpResponse.hasOwnProperty('statusCode') ? httpResponse.statusCode : '[unknown]') + ' in ' + (Date.now() - requestSent) + ' ms');
					cache[linkedContentUrl] = true;
					requestLinkedContent(linkedContentUrls.slice(1), done);
				});
			} else {
				log('http', 'Linked content already cached: ' + linkedContentUrl);
				requestLinkedContent(linkedContentUrls.slice(1), done);
			}
		} else {
			done();
		}
	};

	// Start a random workflow.
	startWorkflow = function () {
		if (config.tuning.clearCachePerWorkflow) {
			cache = {};
		}
		lastRequest = null;
		requestIndex = 0;
		workflowIndex = Math.floor(Math.random() * config.workflows.length);
		queueRequest();
	};

	// Apply missing defaults to config, then queue the first request, others will be called recursively.
	config = _.extend({}, defaults, config);
	testStarted = Date.now();
	startWorkflow();

}(require('underscore'), require('request'), require(process.argv[2])));

To run, for example to start 20 clients with a delay of 5 seconds between each (in bash):

for i in `seq 20` ; do node loadtest.js ./data-example.json > output_`date +%Y%m%d_%H%M%S`_$i.log & sleep 5 ; done

More complete instructions for getting it running:

1. Install node via nvm (see https://github.com/creationix/nvm), current version of node is 0.10.22
2. You will need to do a ". ~/.profile" to get nvm on your path (or logout and login)
3. Install "underscore" and "request" packages via npm (npm install underscore ; npm install request)
4. Put the above script and the configuration file (data-[projectname].json) into the same directory
5. Modify the configuration file to use the correct domain name or IP address as appropriate (local, dev, uat..)
6. Run it: node loadtest.js ./data-[projectname].json

This will ouptut directly to the terminal. When this is working, you can try running multiple instances.