Skip to content

Instantly share code, notes, and snippets.

@zyxue zyxue/execute.py
Created Jun 29, 2016

Embed
What would you like to do?
def execute(cmd, flag_file=None, msg_id='', debug=False):
"""
# http://stackoverflow.com/questions/1606795/catching-stdout-in-realtime-from-subprocess
:param cmd: should never inlcude pipe or redirection, which would requires
a new shell process
This execute logs all stdout and stderr, which could look funny, especially
when it comes to tools like aspc and wget
"""
logger.info('executing: {0}'.format(cmd))
# todo: should check whether cmdsp includes pipe or redirection here
if debug:
return
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
executable='/bin/bash',
# https://docs.python.org/3.5/library/subprocess.html#subprocess.Popen.communicate
universal_newlines=True
)
ioselect(proc)
returncode = proc.returncode
msg = 'returncode: {returncode}. CMD: "{cmd}"'.format(**locals())
if msg_id:
msg = '{0}: {1}'.format(msg_id, msg)
if returncode != 0:
logger.error(msg)
else:
logger.info(msg)
if flag_file is not None:
touch(flag_file, cmd)
return returncode
except OSError as err:
except_msg = ('failed to start, raising OSError {err}. '
'CMD: "{cmd}"'.format(**locals()))
if msg_id:
except_msg = '{0}: {1}'.format(msg_id, except_msg)
logger.exception(except_msg)
def ioselect(proc):
"""
select in the context io completion,
https://docs.python.org/2/library/select.html
"""
while True:
ret = select.select([proc.stdout.fileno(),
proc.stderr.fileno()],
[], [])
# print(ret)
for fd in ret[0]:
if fd == proc.stdout.fileno():
line = proc.stdout.readline()
if line:
# rstrip: remove newline character since logger will
# add one automatically
logger.info('stdout: ' + line.rstrip())
if fd == proc.stderr.fileno():
line = proc.stderr.readline()
if line:
logger.warning('stderr: ' + line.rstrip())
# check if child process has terminated
# https://docs.python.org/3/library/subprocess.html#subprocess.Popen.poll
if proc.poll() != None:
# flush everything first, otherwise, some stdout/stderr may not
# be able to be written to screen (e.g. cutadapt)
for fd in ret[0]:
if fd == proc.stdout.fileno():
for line in proc.stdout.readlines():
if line:
# rstrip: remove newline character since logger will
# add one automatically
logger.info('stdout: ' + line.rstrip())
if fd == proc.stderr.fileno():
for line in proc.stderr.readlines():
if line:
logger.info('stderr: ' + line.rstrip())
break
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.