jsonThreadStreamer.js

//depends on https://github.com/padolsey/operative

var jsonThreadedStreamer = (function (operative) {
	var i = 0,
		threads = [];

	function thread() {
		var t = threads[i],
			limit = thread.limit;

		if (t === undefined) {
			t = threads[i] = thread.create();
		} else {
			t = threads[i];
		}

		i++;
		if (i > limit) {
			i = 0;
		}

		return t;
	}

	thread.limit = 10;

	thread.create = function() {
		var t = operative({
			stream: function(location, urls, callback) {
				var i = 0,
					max = urls.length,
					getting = [];

				if (typeof urls === 'string') {
					getting.push(location + urls);
				} else {
					for (; i < max; i++) {
						getting.push(location + urls[i]);
					}
				}

				Promise
					.all(getting)
					.then(function(jsons) {
						var i = 0,
							j,
							item,
							group,
							groups = jsons,
							iMax = groups.length,
							jMax;

						for(;i<iMax;i++) {
							group = JSON.parse(groups[i]);
							jMax = group.length;
							for(j = 0; j < jMax; j++) {
								item = group[j];
								callback('item', JSON.stringify(item));
							}
						}

						callback('done', true);
					}, function(err) {
						callback('error', err);
					});
			});

		t.stash = [];
		t.busy = false;

		return t;
	};

	thread.kill = function() {
		var i = 0,
			max = threads.length;

		for(;i < max; i++) {
			threads[i].terminate();
		}
	};

	return thread;
})(window.operative);