execute.py

def execute(cmd, flag_file=None, msg_id='', debug=False):
    """
    # http://stackoverflow.com/questions/1606795/catching-stdout-in-realtime-from-subprocess
    :param cmd: should never inlcude pipe or redirection, which would requires
    a new shell process
    This execute logs all stdout and stderr, which could look funny, especially
    when it comes to tools like aspc and wget
    """
    logger.info('executing: {0}'.format(cmd))
    # todo: should check whether cmdsp includes pipe or redirection here

    if debug:
        return
    try:
        proc = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True,
            executable='/bin/bash',
            # https://docs.python.org/3.5/library/subprocess.html#subprocess.Popen.communicate
            universal_newlines=True
        )
        ioselect(proc)
        returncode = proc.returncode
        msg = 'returncode: {returncode}. CMD: "{cmd}"'.format(**locals())
        if msg_id:
            msg = '{0}: {1}'.format(msg_id, msg)

        if returncode != 0:
            logger.error(msg)
        else:
            logger.info(msg)
            if flag_file is not None:
                touch(flag_file, cmd)
        return returncode
    except OSError as err:
        except_msg = ('failed to start, raising OSError {err}. '
                      'CMD: "{cmd}"'.format(**locals()))
        if msg_id:
            except_msg = '{0}: {1}'.format(msg_id, except_msg)
        logger.exception(except_msg)


def ioselect(proc):
    """
    select in the context io completion,
    https://docs.python.org/2/library/select.html
    """
    while True:
        ret = select.select([proc.stdout.fileno(),
                             proc.stderr.fileno()],
                            [], [])
        # print(ret)
        for fd in ret[0]:
            if fd == proc.stdout.fileno():
                line = proc.stdout.readline()
                if line:
                    # rstrip: remove newline character since logger will
                    # add one automatically
                    logger.info('stdout: ' + line.rstrip())
            if fd == proc.stderr.fileno():
                line = proc.stderr.readline()
                if line:
                    logger.warning('stderr: ' + line.rstrip())

        # check if child process has terminated
        # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.poll
        if proc.poll() != None:
            # flush everything first, otherwise, some stdout/stderr may not
            # be able to be written to screen (e.g. cutadapt)
            for fd in ret[0]:
                if fd == proc.stdout.fileno():
                    for line in proc.stdout.readlines():
                        if line:
                            # rstrip: remove newline character since logger will
                            # add one automatically
                            logger.info('stdout: ' + line.rstrip())
                if fd == proc.stderr.fileno():
                    for line in proc.stderr.readlines():
                        if line:
                            logger.info('stderr: ' + line.rstrip())
            break