Skip to content

Instantly share code, notes, and snippets.

@goldenice
Last active November 14, 2015 08:38
Show Gist options
  • Save goldenice/d81a98123806e3cc6c89 to your computer and use it in GitHub Desktop.
Save goldenice/d81a98123806e3cc6c89 to your computer and use it in GitHub Desktop.
Generic self contained 'concurrent' worker in javascript
var request = require('request');
var genericMultiWorker = require('./genericmultiworker.js');
var TARGET_URL = 'http://example.com/';
// Example: POSTing to a URL with a maximum of 5 concurrent requests
var addToWorker = genericMultiWorker(5, function(obj, callback) {
request.post(TARGET_URL, obj, callback);
});
// Example: populate queue with post request bodies
for (var i = 0; i < 1000; i++) {
addToWorker({ example: 'content' });
}
/**
* Generic 'concurrent' worker in javascript
* @param {Number} Maximum simultaneous running functions
* @param {Function} Function to use
* @return {Function} Adds an item to the queue of things being processed
*/
function genericMultiWorker(maxSimultaneous, func) {
var queue = [];
var pending = 0;
function startIfFree() {
if (pending >= maxPending || queue.length == 0) return;
pending++;
func(queue.shift(), function() {
pending--;
startIfFree();
});
}
return function(item) {
queue.push(item);
startIfFree();
}
}
module.exports = genericMultiWorker;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment