read-lines-from-subprocess.py

#!/usr/bin/python
"""Read 10M lines from a subprocess stdout.

   Write to the subprocess' stdin periodically.
"""
import sys
from subprocess import Popen, PIPE
from textwrap import dedent

def main():
    p = Popen([sys.executable, "-c", dedent("""
    import itertools, sys
    for i in itertools.count():
        print(i)
        if i % 1000000 == 0: # read input every 1M lines
           sys.stdout.flush() # make sure the prompt (1M line) is sent
           print(sys.stdin.readline().strip()[::-1]) # reverse line
    """)], stdin=PIPE, stdout=PIPE, bufsize=1)
    for line in iter(p.stdout.readline, b''):
        i = int(line)
        if i % 1000000 == 0: # write to the subprocess every 1M lines
           p.stdin.write(b'abc\n')
           p.stdin.flush()
           response = p.stdout.readline()
           if response.strip() != b'cba': # should receive reversed line
               sys.stderr.write('error %d %r' % (i, response,))
           sys.stderr.write('.')
           sys.stderr.flush()
           if i == 10*1000000: # 10M lines
               break
    p.stdin.close()
    p.stdout.close()
    p.terminate()
    p.wait()

main()