Created
July 30, 2020 11:08
-
-
Save wallyhall/0e1f0a478e1af9f47e51e43bb7748b3c to your computer and use it in GitHub Desktop.
Parallel processing in bash, collecting all resulting STDOUT from children
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# trap handler to cleanly shutdown all children if aborted (i.e. ^C is invoked) | |
function abort { | |
cd - > /dev/null | |
# ignore the SIGTERM that is about to be issued | |
trap '' SIGTERM | |
echo "Signalling shutdown to all children..." | |
kill 0 | |
echo "Waiting on all children to finish..." | |
wait | |
exit 1 | |
} | |
# usage message | |
if [ $# -lt 2 ]; then | |
echo "Usage: $(basename $0) <file> <children>" >&2 | |
exit 1 | |
fi | |
# number of children argument | |
if ! [[ $2 =~ ^[0-9]+$ ]]; then | |
echo "Second argument must be number of parallel children" >&2 | |
exit 1 | |
fi | |
CHILDREN=$2 | |
# input file argument (for demo purposes) | |
INPUTFILE="$(readlink -m "$1")" | |
if [ ! -r "$INPUTFILE" ]; then | |
echo "Could not read input file '$INPUTFILE'" >&2 | |
exit 1 | |
fi | |
# calculate how to split the input file up per-child (for demo purposes) | |
echo "Processing '$INPUTFILE'..." | |
FILELEN=$(wc -l "$INPUTFILE" | cut -d ' ' -f 1) | |
if [ $FILELEN -eq 0 ]; then | |
echo "File length is 0, nothing to do." | |
exit 1 | |
fi | |
echo "File length is $FILELEN" | |
if [ $FILELEN -lt $CHILDREN ]; then | |
CHILDREN=$FILELEN | |
echo "Limiting children count to $FILELEN" | |
fi | |
let PERCHILD=FILELEN/CHILDREN | |
echo "$PERCHILD to be ran per each child" | |
# can customise to pipe output into something else or a variable even | |
OUTCMD='tee combined.out' | |
# parallel children processing | |
trap abort SIGINT | |
{ | |
for i in $(seq 1 $CHILDREN); do | |
let HEAD=PERCHILD*i | |
( | |
echo "Spawned child $i" >&2 | |
for FILELINE in $(head $INPUTFILE -n $HEAD | tail -n $PERCHILD); do | |
echo "[INFO] Child $i processing '$FILELINE'" >&2 | |
sleep 0.2 # in reality some real processing would happen here... | |
if [ $? -ne 0 ]; then | |
echo "[WARN] Child $i failed to process '$FILELINE'" >&2 | |
else | |
echo "Child $i processed '$FILELINE'" | |
fi | |
done | |
exit | |
) & | |
done | |
} | eval "$OUTCMD" || abort | |
# cleanup at end | |
echo "Waiting on children..." | |
wait | |
exit 0 | |
# end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment