Randomly shuffle a newline-delimited file that's larger than main memory

chunked_shuffle.sh

set -e

if [ "$#" -lt 3 ]; then
    echo "Usage: chunked_shuffle filename parts outfile"
    exit 1
fi

filename=$1
parts=$2
outfile=$3

awk -v parts=$parts -v filename=$filename 'BEGIN{srand();} { print > filename"."int(rand() * parts) }' $filename

tmp_outfile=$filename.out
> $tmp_outfile

for i in $(seq 0 $[$parts - 1]); do
    shuf $filename.$i >> $tmp_outfile
    rm $filename.$i
done

mv $tmp_outfile $outfile