Skip to content

Instantly share code, notes, and snippets.

@schellingb
Last active May 4, 2020 13:43
Show Gist options
  • Save schellingb/1077d6976c6ee8494e0b2970d90c2df8 to your computer and use it in GitHub Desktop.
Save schellingb/1077d6976c6ee8494e0b2970d90c2df8 to your computer and use it in GitHub Desktop.
Run a maximum of X processes at the same time in a non-async node script
const fs = require('fs'), child_process = require('child_process');
function RunSync(cmd, args)
{
console.log('Running sync: ' + cmd + ' ' + args.join(' '));
const proc = child_process.spawnSync(cmd, args, {stdio:[0,1,2]});
if (proc.status === null) throw 'Error while starting ' + cmd + '. Executable not found at path or no access.';
if (proc.status !== 0) throw 'Error while running ' + cmd + '. An error should have been printed above.';
}
function WaitProcs(procs, maxProcs)
{
for (;;) //clear finished processes and wait while there are more than maxProcs running at the same
{
for (var i = procs.length; i--;)
{
if (procs[i].IsRunning()) continue;
procs[i].AbortIfError();
procs.splice(i, 1);
}
if (procs.length <= (maxProcs|0)) return;
child_process.spawnSync(process.execPath,['-e','setTimeout(function(){},100)']); //sleep 100 ms
}
}
function RunAsync(cmd, args, outPath, procs, maxProcs)
{
WaitProcs(procs, maxProcs - 1);
console.log('Running async: ' + cmd + ' ' + args.join(' '));
var proc = child_process.spawn(cmd, args, {stdio:[0,1,2]});
if (proc.pid === undefined) throw 'Error while starting ' + cmd + '. Executable not found at path or no access.';
procs.push({
AbortIfError: () => {if (!fs.existsSync(outPath)) throw 'Error while running ' + cmd + '. An error should have been printed above.'},
IsRunning: () => {try{return process.kill(proc.pid,0)}catch{}},
});
}
// fake program that waits for a while (arg 1) then writes a file (arg 2)
var myCmd = process.execPath;
var myArgs = [ '-e', "setTimeout(function(){require('fs').writeFileSync(process.argv[2], 1)},process.argv[1])" ];
// spawn 10 processes that take 2 seconds async, except the last one which takes .5 seconds and is spawned synchronous
var maxProcs = 4; //run a maximum of 4 processes at the same time
var procs = [];
for (var i = 0, iMax = 10; i != iMax; i++)
{
var waitTime = (i == iMax - 1 ? 500 : 2000);
var outPath = "TEMP" + i + ".DAT";
try { fs.unlinkSync(outPath); } catch (e) {}
(i == iMax - 1 ? RunSync : RunAsync)(myCmd, myArgs.concat([ waitTime, outPath ]), outPath, procs, maxProcs);
}
console.log("Waiting for remaining processes ...");
WaitProcs(procs);
console.log("Done!");
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment