process.py

from subprocess import Popen, PIPE

outproc = None
linecount = 0
p = Popen('cat input.txt', shell=True, stdout=PIPE)
for line in p.stdout.xreadlines():
    if linecount % 1000000 == 0:
        outfile = "output%03d" %(linecount // 1000000, )
        if outproc:
        	#outproc.stdin.flush()
            outproc.stdin.close()
            result = outproc.wait() # <-- deadlock here
            assert result == 0, "outproc exited with %s" %(result, )
        outproc = Popen('cat > %s'%outfile, shell=True, stdin=PIPE)
                        
    linecount += 1
    outproc.stdin.write(line)
# p.stdout.close() <-- you can't close p.stdout
result = p.wait()
assert result == 0, "p exited with %s" %(result, )

setup.py

from subprocess import Popen, PIPE

# create a dummy input file (10M lines)
f = open('input.txt', 'w')
for i in xrange(10000000):
    f.write('%s\n'%('%d'%i)[-1])
f.close()

    p = Popen('cat > myfile.txt', shell=True, stdin=PIPE)
p.stdin.write('hello')
p.stdin.flush()
p.stdin.close()