sh-machine.py

from grain.delayed import delayed, each, run
from grain.util import QueueLimiter
from grain.resource import Node

import trio
from pathlib import Path
from functools import partial

@delayed
async def execjob(cmd):
    await trio.run_process(cmd, shell=True)

async def clear_dir(base, glob):
    async with QueueLimiter(300) as ql:
        for f in Path(base).glob(glob):
            await ql(execjob @ Node(N=1,M=4))(f'sox {f} -r 16000 /N/slate/zhanghar/ntc/wavs-16k/{f.name}')
            #^^^wait for submission, not completion
    #^^^ql scope will not end until all jobs are done

async def clear_dir_nolimit(base, glob):
    all_tasks = []
    for f in Path(base).glob(glob):
        all_tasks += [(execjob @ Node(N=1,M=4))(f'sox {f} -r 16000 /N/slate/zhanghar/ntc/wavs-16k/{f.name}')]
    await each(all_tasks)
    
run(
    partial(clear_dir, '/N/slate/zhanghar/ntc/wavs', '*.wav'),
    nolocal=True,
)