huntergatherer.js

var sys = require('sys');
var url = require('url');
var oauth = require('oauth');
var request = require('request');
var querystring = require('querystring');
var Extendable = require("extendables").Extendable;


var defaultOptions = {
	url: '',
	method: 'GET',
	headers: {},
	body: '',
	limitKey: 'limit',
	offsetKey: 'offset',
	offsetCallback: function(limit, count) {
		return limit;
	},
	startOffset: 0,
	pool: {
		maxSockets: 10
	}
};

exports.gather = function(userOptions, countCallback, dataCallback) {
	var mergedOptions = Extendable.extend(defaultOptions);
	mergedOptions = mergedOptions.extend(userOptions);
	
	var options = new mergedOptions();
	
	request(options, function(error, response, body) {
		if (error || response.statusCode >= 300) {
			console.log('Request failed for URL: ' + options.url + '(' + error + ')');
		}
		
		// process total count and initial request data
		var count = countCallback(error, response, body);
		dataCallback(error, response, body);
		
		var uri = url.parse(options.url, true);
		
		// now parallelize the hell out of the rest
		var increment = +options.offsetCallback(uri.query[options.limitKey]);
		for (var i = +options.startOffset; i < count; i += increment) {
			//var url = 'http://openapi.etsy.com/v2/sandbox/private/listings/active?limit=100&offset='+i+'&keywords=owl&sort_on=created';
			
			var queryString = url.parse(options.url, true);
			delete queryString.search;

			queryString.query[options.offsetKey] = i;
			options.url = url.format(queryString);
			
			makeRequest(options, options.url, dataCallback);
		}
		
		return;
	});
};

function makeRequest(options, url, dataCallback) {
    request(options, function(error, response, body) {
		if (error || response.statusCode >= 300) {
			console.log('Request failed for URL: ' + url + '(' + error + ')');
			return dataCallback(error);
		}
		console.log('Calling dataCallback for URL: ' + url);
		dataCallback(error, response, body);
	});
}

test-huntergatherer.js

(typeof define !== 'function' ? function($){ $(require, exports, module); } : define)(function(require, exports, module, undefined) {

exports['test callbacks'] = function(assert, done) {

	// fire up a test server to test against
	var http = require('http');
	var server = http.createServer(function(request, response) {
		var body = [];
		var url = require('url').parse(request.url, true);

		var offset = parseInt(url.query.offset);
		var limit = parseInt(url.query.limit);

		for (var i = offset; i < offset + limit; i++) {
			body.push(i);
		}
		var data = {
			count: Object.keys(body).length,
			results: body
		}


		response.statusCode = 200;
		response.setHeader('Content-Type', 'application/json');
		response.write(JSON.stringify(data));
		response.end();
	});
	server.listen(9876);


	var hg = require('../lib/huntergatherer');
	var options = {
		url: 'http://localhost:9876/?limit=10&offset=0',
		limitKey: 'limit',
		offsetKey: 'offset',
		limit: 10,
		startOffset: 0
	};

	var callbackCount = 0;
	hg.gather(options,
			function(error, response, data) {
				assert.equal(error, null, 'Successfully retrieved count from remote server');
				return 100;
			},
			function(error, response, data) {
				assert.equal(error, null, 'Successfully processing data from remote server');

				var data = JSON.parse(data);
				assert.equal(data.results.length, 10, 'Received the correct number of results');
				callbackCount++;
				if (callbackCount == 11) {
					done();
					server.close();
				}
			}
	);
};

if (module == require.main)
  require('test').run(exports);

});