parallel workflow orchestration using wait-cmd in shellscript #bash #parallel #shellscript #speedupCI #windmill #airflow

workflow.sh

#!/bin/bash
#
# Q: how can I track & run tasks in parallel (speed up CI pipeline e.g.)
#    confidently without introducing platforms like Apache
#    Airflow or Windmilll?
#
# A: the 'wait' command in shellscript
#
set -eE && trap "echo '❌ halting flow'" ERR
run(){ time "$@" 2>&1 | sed "s/^/[$1] /g" | tee .log; }

run ./mycmd    &
run ./othercmd &

wait
grep -q ❌ .log && exit 1 || exit 0

# OUTPUT:
# 
# [./othercmd] dolor sit amet
#   real    0m2.010s
#   user    0m0.012s
#   sys     0m0.018s
# [./mycmd] lorem ipsum
#   real    0m3.010s
#   user    0m0.012s
#   sys     0m0.018s
# shell returned 0