Parallel stuff on the shell with gnu parallel utility

optimize-images.sh

#!/usr/bin/env bash
## You can use this to crush images in parallel via the shell alone.
## Make sure the GNU parallel utility is installed:
## - apt-get install parallel
## - yum install parallel

## For a simple example, run the following to simulate with echo and sleep:
## ls --color=no /usr/share/doc | parallel -j 16 --gnu -i -- 'sh -c "sleep 0.5 && echo {}"'
##   (-i tells parallel to substitute a filename when it sees {} in the command, rather than
##    simply appending it to the command, as it would normally do if you omit -i)


OPTIMIZER=imagecrusher
IMAGES_DIR=/www/site/pub/static/images

## Spawn up to this number of jobs
MAX_JOBS=16

## Don't spawn any more jobs if load average is above this threshold
MAX_LOAD=4


## Below, anything after the -- is the command `parallel` will run for each
## file piped to it.

ls --color=no $IMAGES_DIR | parallel --gnu -j $MAX_JOBS -l $MAX_LOAD -- $OPTIMIZER