Parallel processing in Craft with parallel

gistfile1.txt

#!/bin/bash
LOCAL_PARALLEL_JOBS=$1
PARALLEL_JOBS=${LOCAL_PARALLEL_JOBS:-4}

# parallel creates a subshell, which will use the default $SHELL
# This makes sure php runs paralell with the correct shell
export SHELL=/bin/bash

index() {
    echo Indexing $1
    # echo `craft/app/etc/console/yiic elasticsearch test --id=$1`
    craft/app/etc/console/yiic elasticsearch mapEntry --id=$1
    echo Done with $1
}

export -f index

# Make sure logs directory exists
if [ ! -d "logs" ]; then
    mkdir logs
fi

# Save ids to txt file
craft/app/etc/console/yiic elasticsearch saveEntryIds

# Map entry content to json files
cat data/entries/ids.txt | parallel --progress --eta --joblog logs/parallel-es-indexing.log -j $PARALLEL_JOBS index

# Index json files
craft/app/etc/console/yiic elasticsearch indexJsonFiles