redis_failover.sh

# Make the eventual new master a slave of the old master
knife node run_list add $NEW_MASTER 'role[redis-slave]'
ssh $NEW_MASTER sudo chef-client

# Wait a bit for the slave to catch up
sleep 30

# Designate the slave as the new master and remove the master status from the
# old master. This only updates chef metadata, the changes won't take place
# until a chef-client is run.
knife node run_list remove $NEW_MASTER 'role[redis-slave]'
knife node run_list add    $NEW_MASTER 'role[redis-master]'
knife node run_list remove $OLD_MASTER 'role[redis-master]'

# Run chef-client on the redis clients to point them to the new master
# The new master is still a slave of the old master at this point so there's a
# transition period where clients are writing to both redis's, with writes
# to the old master getting replicated to the new master
cap chef:dogweb-frontend

sleep 5

# Run chef-client on the redis hosts to remove the replication link between
# the old master and new master
HOSTS="$NEW_MASTER,$OLD_MASTER" cap invoke COMMAND="sudo chef client"

ssh $OLD_MASTER sudo stop redis-server