bash: Run parallel commands and fail if any of them fails

parallel

#! /usr/bin/env bash
#
# Run parallel commands and fail if any of them fails.
#

set -eu

pids=()

for x in 1 2 3; do
  ls /not-a-file &
  pids+=($!)
done

for pid in "${pids[@]}"; do
  wait "$pid"
done

parallel-explained

#! /usr/bin/env bash
#
# Run parallel commands and fail if any of them fails.
#
# The expected output is something like this:
#
#   $ ./parallel-explained
#   ls: cannot access '/not-a-file': No such file or directory
#   ls: cannot access '/not-a-file'ls: cannot access '/not-a-file': No such file or directory
#   : No such file or directory
#
# Our 'parallel-explained' script exited with code '2', because it's the exit
# code of one of the failed 'ls' jobs:
#
#   $ echo $?
#   2
#

# 'set -e' tells the shell to exit if any of the foreground command fails,
# i.e. exits with a non-zero status.
set -eu

# Initialize array of PIDs for the background jobs that we're about to launch.
pids=()

for x in 1 2 3; do
  # Run a command in the background. We expect this command to fail.
  ls /not-a-file &

  # Add the PID of this background job to the array.
  pids+=($!)
done

# Wait for each specific process to terminate.
# Instead of this loop, a single call to 'wait' would wait for all the jobs
# to terminate, but it would not give us their exit status.
#
for pid in "${pids[@]}"; do
  #
  # Waiting on a specific PID makes the wait command return with the exit
  # status of that process. Because of the 'set -e' setting, any exit status
  # other than zero causes the current shell to terminate with that exit
  # status as well.
  #
  wait "$pid"
done

Here's an improved version, which tries to terminate the remaining children before exiting:

#! /usr/bin/env bash
#
# Run parallel commands and fail if any of them fails.
#
set -eu

pids=()

(sleep 3; echo ok3) &
pids+=($!)

(sleep 2; echo fail2; exit 1) &
pids+=($!)

(sleep 1; echo ok1) &
pids+=($!)

for pid in "${pids[@]}"; do
  if wait -n; then
    :
  else
    status=$?
    echo "One of the subprocesses exited with nonzero status $status. Aborting."
    for pid in "${pids[@]}"; do
      # Send a termination signal to all the children, and ignore errors
      # due to children that no longer exist.
      kill "$pid" 2> /dev/null || :
    done
    exit "$status"
  fi
done

It's a little complicated and maybe incorrect in some respects.

Here's my solution:

#!/usr/bin/env bash

set -eu

ARG1=${1:-$(nproc --ignore=1)}

pids=()

for x in $(seq 1 ${ARG1}); do
  python3 unit_tests.py &
  pids+=($!)
done

for pid in "${pids[@]}"; do
  if wait -n; then
    :
  else
    exit_code=$?
    echo "Process exited with $exit_code, killing other tests now."
    for pid in "${pids[@]}"; do
      kill -9 "$pid" 2> /dev/null || :
    done
    exit "$exit_code"
  fi
done