Celery + Redis trickle backlog

gistfile1.sh

function trickle_backlog() {
    queue=$1
    backlog=$2
    max_depth=$3
    host=$4
    db=$5
    while [ $(redis-cli -h $host -n $db llen $backlog) -gt 0 ]; do
	while [ $(redis-cli -h $host -n $db llen $queue) -gt $max_depth ]; do
	    echo "snoozing..."
	    sleep 1
	done
	for i in {1..100}; do 
	    redis-cli -h $host -n $db RPOPLPUSH $backlog $queue > /dev/null
	done
	date
	echo "pushed 100"
    done
}

host=...
queue=mqueue_reqs2
q2=mqueue_reqs2_backlog
n=100
db=0

# rename the existing queue
redis-cli -h $host -n 1 rename $queue $q2

# run the trickler in a screen
trickle_backlog $queue $q2 $n $host $db