Run an rsync as an asyncio subprocess and stream the progress output

async_rsync_progress.py

"""Demo for streaming rsync progress from a subprocess with asyncio

Before running:

    dd if=/dev/zero of=./1000mb.bin bs=$((1024 ** 2)) count=1000
    mkdir tmp
"""
import asyncio
from subprocess import PIPE

CR = b'\r'


async def noise_task():
    w = 0.65
    for i in range(1000):
        print(f'--- {i} ---')
        await asyncio.sleep(w)


async def cr_reader(stream, read_buf=512):
    buf = b''
    while not stream.at_eof():
        buf += await stream.read(read_buf)
        lines = buf.split(CR)
        if len(lines) == 1:  # if no CR
            continue
        buf = lines.pop()  # '' if we ended with CR, partial line otherwise
        for raw_line in lines:
            yield raw_line.decode('utf-8')


async def test():
    t = asyncio.create_task(noise_task())
    p = await asyncio.create_subprocess_exec('rsync', '-v', '--progress', '1000mb.bin', 'tmp/', stdout=PIPE, stderr=PIPE)

    async for line in cr_reader(p.stdout):
        print('==', line.strip(), '==')

    t.cancel()
    await p.wait()


asyncio.run(test())