Skip to content

Instantly share code, notes, and snippets.

@Phlogistique
Last active October 20, 2016 17:35
Show Gist options
  • Save Phlogistique/adc5b7b00ff0b8fe061480e607cf4984 to your computer and use it in GitHub Desktop.
Save Phlogistique/adc5b7b00ff0b8fe061480e607cf4984 to your computer and use it in GitHub Desktop.
really minimal jobserver
import sys
import subprocess
import queue
import threading
import functools
import multiprocessing
exit = []
def worker(q):
print("Worker spawned", file=sys.stderr)
while not exit:
try:
item = q.get(timeout=0.1)
except queue.Empty:
pass
else:
run(item)
def run(cmd):
print("Running", cmd, file=sys.stderr)
try:
retcode = subprocess.call(cmd, shell=True)
if retcode < 0:
print("Child was terminated by signal", -retcode, file=sys.stderr)
exit.append(1)
elif retcode > 0:
print("Child returned with error", retcode, file=sys.stderr)
exit.append(1)
except OSError as e:
print("Execution failed:", e, file=sys.stderr)
exit.append(1)
try:
njobs = int(sys.argv[1])
except:
njobs = multiprocessing.cpu_count()
print("njobs:", njobs, file=sys.stderr)
if njobs == 0:
threads = [] # threads will be created as needed
elif njobs == 1:
pass # no setup needed
else:
work_items = queue.Queue()
threads = [threading.Thread(target=functools.partial(worker, work_items))
for _ in range(njobs)]
for t in threads:
t.start()
for cmd in sys.stdin:
if njobs == 0:
t = threading.Thread(target=functools.partial(run, cmd))
thread.append(t)
t.start()
elif njobs == 1:
run(cmd)
else:
work_items.put(cmd)
if exit:
break
exit.append(0)
if njobs != 1:
for t in threads:
t.join()
if exit:
sys.exit(exit[0])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment