Run multi processes over Python Thread. example : python myltip.py -c 'sleep 4 && echo ok' -c 'sleep 4 && echo ok'
import Queue | |
import threading | |
import time | |
import sys | |
import os | |
import multiprocessing | |
from optparse import OptionParser | |
import subprocess | |
import json | |
class ThreadCmd(threading.Thread): | |
def __init__(self, cmdQueue, responses): | |
threading.Thread.__init__(self) | |
self.cmdQueue = cmdQueue | |
self.responses = responses | |
def run(self): | |
while True: | |
cmd = self.cmdQueue.get() | |
try: | |
start = time.time() | |
p = subprocess.Popen(cmd, shell=True, bufsize=1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
t = list(p.communicate()) | |
t.insert(0, time.time() - start) | |
t.insert(0, p.returncode) | |
t.insert(0, cmd) | |
self.responses.put(t); | |
#signals to queue job is done | |
self.cmdQueue.task_done() | |
except: | |
self.responses.put(sys.exc_info()[0]); | |
self.cmdQueue.task_done() | |
def run(cmds, n): | |
cmdQueue = Queue.Queue() | |
responses = Queue.Queue() | |
#print "%d threads" % n | |
for i in range(n): | |
t = ThreadCmd(cmdQueue, responses) | |
t.setDaemon(True) | |
t.start() | |
for cmd in cmds: | |
cmdQueue.put(cmd) | |
#wait on the queue until everything has been processed | |
cmdQueue.join() | |
items = [] | |
while True: | |
try: | |
items.append(responses.get_nowait()) | |
except: | |
break | |
return json.dumps(items) | |
def parseOpt(): | |
parser = OptionParser() | |
parser.add_option( | |
"-c", "--command", | |
action="append", | |
dest="commands", | |
help="Command lines being executed", | |
) | |
parser.add_option( | |
"-t", "--threads", | |
dest="threads", | |
type="int", | |
default=multiprocessing.cpu_count(), | |
help="Number of threads need to be created" | |
) | |
(options, args) = parser.parse_args() | |
return options | |
options = parseOpt() | |
#mainStart = time.time() | |
#t = threading.Thread(target=main, args=(options.threads,)) | |
#t.start() | |
#t.join(1) | |
print run(options.commands, options.threads) | |
#print "Elapsed Time: %s" % (time.time() - mainStart) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment