Skip to content

Instantly share code, notes, and snippets.

@xxorax xxorax/multip.py
Created Feb 1, 2013

Embed
What would you like to do?
Run multi processes over Python Thread. example : python myltip.py -c 'sleep 4 && echo ok' -c 'sleep 4 && echo ok'
import Queue
import threading
import time
import sys
import os
import multiprocessing
from optparse import OptionParser
import subprocess
import json
class ThreadCmd(threading.Thread):
def __init__(self, cmdQueue, responses):
threading.Thread.__init__(self)
self.cmdQueue = cmdQueue
self.responses = responses
def run(self):
while True:
cmd = self.cmdQueue.get()
try:
start = time.time()
p = subprocess.Popen(cmd, shell=True, bufsize=1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
t = list(p.communicate())
t.insert(0, time.time() - start)
t.insert(0, p.returncode)
t.insert(0, cmd)
self.responses.put(t);
#signals to queue job is done
self.cmdQueue.task_done()
except:
self.responses.put(sys.exc_info()[0]);
self.cmdQueue.task_done()
def run(cmds, n):
cmdQueue = Queue.Queue()
responses = Queue.Queue()
#print "%d threads" % n
for i in range(n):
t = ThreadCmd(cmdQueue, responses)
t.setDaemon(True)
t.start()
for cmd in cmds:
cmdQueue.put(cmd)
#wait on the queue until everything has been processed
cmdQueue.join()
items = []
while True:
try:
items.append(responses.get_nowait())
except:
break
return json.dumps(items)
def parseOpt():
parser = OptionParser()
parser.add_option(
"-c", "--command",
action="append",
dest="commands",
help="Command lines being executed",
)
parser.add_option(
"-t", "--threads",
dest="threads",
type="int",
default=multiprocessing.cpu_count(),
help="Number of threads need to be created"
)
(options, args) = parser.parse_args()
return options
options = parseOpt()
#mainStart = time.time()
#t = threading.Thread(target=main, args=(options.threads,))
#t.start()
#t.join(1)
print run(options.commands, options.threads)
#print "Elapsed Time: %s" % (time.time() - mainStart)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.