example of how to use ray.io to execute multiple OS processes in parallel (something like GNU parallel is doing)

RayParallelExec.py

# example of how to use ray.io to execute multiple 
# OS processes in parallel (even in multiple machines)
# (something like GNU parallel is doing)

pip install -U "ray[default]"

import ray
import os
import time

os.environ["RAY_DISABLE_MEMORY_MONITOR"] = "1"
ray.init(include_dashboard=True)

# %%

@ray.remote
def remote_exec(i):
    import subprocess
    list_files = subprocess.run(["ls", "-l"], capture_output=True, text=True)
    # print("The exit code was: %d" % list_files.returncode)
    out = list_files.stdout.split('\n')
    return out

futures = [remote_exec.remote(i) for i in range(10)]
print(ray.get(futures))
print("finished")

# %%
ray.shutdown()