Skip to content

Instantly share code, notes, and snippets.

@wallyhall
Created July 30, 2020 11:08
Show Gist options
  • Save wallyhall/0e1f0a478e1af9f47e51e43bb7748b3c to your computer and use it in GitHub Desktop.
Save wallyhall/0e1f0a478e1af9f47e51e43bb7748b3c to your computer and use it in GitHub Desktop.
Parallel processing in bash, collecting all resulting STDOUT from children
#!/bin/bash
# trap handler to cleanly shutdown all children if aborted (i.e. ^C is invoked)
function abort {
cd - > /dev/null
# ignore the SIGTERM that is about to be issued
trap '' SIGTERM
echo "Signalling shutdown to all children..."
kill 0
echo "Waiting on all children to finish..."
wait
exit 1
}
# usage message
if [ $# -lt 2 ]; then
echo "Usage: $(basename $0) <file> <children>" >&2
exit 1
fi
# number of children argument
if ! [[ $2 =~ ^[0-9]+$ ]]; then
echo "Second argument must be number of parallel children" >&2
exit 1
fi
CHILDREN=$2
# input file argument (for demo purposes)
INPUTFILE="$(readlink -m "$1")"
if [ ! -r "$INPUTFILE" ]; then
echo "Could not read input file '$INPUTFILE'" >&2
exit 1
fi
# calculate how to split the input file up per-child (for demo purposes)
echo "Processing '$INPUTFILE'..."
FILELEN=$(wc -l "$INPUTFILE" | cut -d ' ' -f 1)
if [ $FILELEN -eq 0 ]; then
echo "File length is 0, nothing to do."
exit 1
fi
echo "File length is $FILELEN"
if [ $FILELEN -lt $CHILDREN ]; then
CHILDREN=$FILELEN
echo "Limiting children count to $FILELEN"
fi
let PERCHILD=FILELEN/CHILDREN
echo "$PERCHILD to be ran per each child"
# can customise to pipe output into something else or a variable even
OUTCMD='tee combined.out'
# parallel children processing
trap abort SIGINT
{
for i in $(seq 1 $CHILDREN); do
let HEAD=PERCHILD*i
(
echo "Spawned child $i" >&2
for FILELINE in $(head $INPUTFILE -n $HEAD | tail -n $PERCHILD); do
echo "[INFO] Child $i processing '$FILELINE'" >&2
sleep 0.2 # in reality some real processing would happen here...
if [ $? -ne 0 ]; then
echo "[WARN] Child $i failed to process '$FILELINE'" >&2
else
echo "Child $i processed '$FILELINE'"
fi
done
exit
) &
done
} | eval "$OUTCMD" || abort
# cleanup at end
echo "Waiting on children..."
wait
exit 0
# end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment