How to use two Cloud Servers to make a decently available load balancer

failover-lb-member.sh

#!/bin/sh

# Failover a member of a load balancer pair.  If this Cloud Server is
# currently the primary, it becomes the secondary, and vice versa.

#/ Usage: failover-lb-member

set -e

usage() {
    grep "^#/" "$0" | cut -c"4-" >&2
    exit "$1"
}
while [ "$#" -gt 0 ]
do
    case "$1" in
        -h|--help) usage 0;;
        -*) usage 1;;
        *) break;;
    esac
done

# Primary to secondary.
if ip addr show "eth0" | grep -q "inet 1.2.3.4/24"
then
    ip addr del "1.2.3.4/24" dev "eth0"
    ip addr add "5.6.7.8/24" dev "eth0"
    route add default dev "eth0" gw "5.6.7.1"
    arping -A -I"eth0" -c"3" "5.6.7.8"

# Secondary to primary.
else
    ip addr del "5.6.7.8/24" dev "eth0"
    ip addr add "1.2.3.4/24" dev "eth0"
    route add default dev "eth0" gw "1.2.3.1"
    arping -A -I"eth0" -c"3" "1.2.3.4"

fi

failover-lb-pair.sh

#!/bin/sh

# Failover a load balancer pair.  The primary becomes the secondary and vice
# versa.  If one is down hard it doesn't prevent the other from proceeding.

#/ Usage: failover-lb-pair <address> <address>
#/   <address> private IP address of a Cloud Server in the load balancer pair

set -e

usage() {
    grep "^#/" "$0" | cut -c"4-" >&2
    exit "$1"
}
while [ "$#" -gt 0 ]
do
    case "$1" in
        -h|--help) usage 0;;
        -*) usage 1;;
        *) break;;
    esac
done
[ "$1" -a "$2" ] || usage 1

for "$ADDRESS" in "$1" "$2"
do ssh "$ADDRESS" sudo failover-lb-member &
done
wait

provision-lb-pair.sh

#!/bin/sh

# Provision a pair of Cloud Servers to act as redundant load balancers.  Their
# public IP addresses are shared so traffic can be failed over from one to the
# other without involving the DNS.
#
# HERE BE DRAGONS: Sharing a Cloud Server's public IP address with a Shared IP
# Group via the API doesn't actually work.  You have to call Rackspace support
# and have them mark the public IP addresses as shared.  They're happy to do
# it and it only has to happen once per Cloud Server, so it's not really that
# big of a deal.
#
# This program requires <https://github.com/rcrowley/json.sh>.

#/ Usage: provision-lb-pair -u<rackspace-username> -k<rackspace-api-key> <name>
#/   -u<rackspace-username> Rackspace Cloud username
#/   -k<rackspace-api-key>  Rackspace Coud API key
#/   <name>                 name to use for the load balancer pair

RACKSPACE_IDENTITY_URL="https://lon.identity.api.rackspacecloud.com/v1.0"

RACKSPACE_FLAVOR_256="1"
RACKSPACE_IMAGE_ONEIRIC="119"

set -e

usage() {
    grep "^#/" "$0" | cut -c"4-" >&2
    exit "$1"
}
while [ "$#" -gt 0 ]
do
    case "$1" in
        -u) RACKSPACE_USERNAME="$2" shift 2;;
        -u*) RACKSPACE_USERNAME="$(echo "$1" | cut -c"3-")" shift;;
        -k) RACKSPACE_API_KEY="$2" shift 2;;
        -k*) RACKSPACE_API_KEY="$(echo "$1" | cut -c"3-")" shift;;
        -h|--help) usage 0;;
        -*) usage 1;;
        *) break;;
    esac
done
NAME="$1"
[ "$RACKSPACE_USERNAME" -a "$RACKSPACE_API_KEY" -a "$NAME" ] || usage 1

TMP="$(mktemp)"
trap "rm -f \"$TMP\"" EXIT INT QUIT TERM

# Trade a Rackspace API key for an access token.
curl -D"$TMP" -H"X-Auth-Key: $RACKSPACE_API_KEY" -H"X-Auth-User: $RACKSPACE_USERNAME" -f -s "$RACKSPACE_IDENTITY_URL"
while read HEADER
do
    case "$HEADER" in
        "X-Auth-Token:"*) TOKEN="$(echo "$HEADER" | cut -d" " -f"2-" | tr -d "\\r")";;
        "X-Server-Management-Url:"*) URL="$(echo "$HEADER" | cut -d" " -f"2-" | tr -d "\\r")";;
    esac
done <"$TMP"

# Create an empty Shared IP Group.
curl -H"Content-Type: application/json" \
     -H"X-Auth-Token: $TOKEN" \
     -X"POST" \
     -d"{\"sharedIpGroup\":{\"name\":\"$NAME\"}}" \
     -s \
     "$URL/shared_ip_groups" |
tee "$TMP"
echo
SHARED_IP_GROUP="$(cat "$TMP" | json.sh | grep "^/sharedIpGroup/id" | cut -d" " -f"3-")"

# Create a pair of Cloud Servers.
for LETTER in "a" "b"
do

    # Create a Cloud Server that's a member of the Shared IP Group.
    curl -H"Content-Type: application/json" \
         -H"X-Auth-Token: $TOKEN" \
         -X"POST" \
         -d"{\"server\":{\"flavorId\":$RACKSPACE_FLAVOR_256,\"imageId\":$RACKSPACE_IMAGE_ONEIRIC,\"name\":\"$NAME$LETTER\",\"sharedIpGroupId\":$SHARED_IP_GROUP}}" \
         -s \
         "$URL/servers" |
    tee "$TMP"
    echo
    SERVER="$(cat "$TMP" | json.sh | grep "^/server/id" | cut -d" " -f"3-")"
    ADDRESS="$(cat "$TMP" | json.sh | grep "^/server/addresses/public/0" | cut -d" " -f"3-")"

    # Wait for the Cloud Server to build.
    while true
    do
        curl -H"Content-Type: application/json" \
             -H"X-Auth-Token: $TOKEN" \
             -s \
             "$URL/servers/$SERVER" |
        json.sh |
        grep "^/server/status string ACTIVE\$" &&
        break
        sleep 10
    done

    # Share the newly built Cloud Server's public IP address with the other
    # members of the Shared IP Group.  This doesn't actually work and instead
    # you have to call Rackspace support and have them do it.
    curl -H"Content-Type: application/json" \
         -H"X-Auth-Token: $TOKEN" \
         -X"PUT" \
         -d"{\"shareIp\":{\"sharedIpGroupId\":$SHARED_IP_GROUP}}" \
         -s \
         "$URL/servers/$SERVER/ips/public/$ADDRESS"
    echo

done