fix-mongo-cluster.sh

#!/bin/bash
# Manually setup juju mongodb clustering when failed from unresolvable hostnames,
# eg when using MaaS + LXC provisioning, see lp#1274947
set -u
SERVICE=${1:-mongodb}
echo "*** $SERVICE: checking clustered units..."
replset=$(juju get $SERVICE| python -c 'import yaml, sys;print yaml.load(sys.stdin)["settings"]["replicaset"]["value"]')
units=$(juju status $SERVICE | \
    python -c 'import yaml, sys;print " ".join(sorted(yaml.load(sys.stdin)["services"]["'$SERVICE'"]["units"].keys()))')
# set positional parameters to $units, for easier handling (shift, $1, $2, etc)
set -- $units
echo "units: $*"
# Select 1st as master
master=$1
n_total=$#
shift
rs_ok=$(juju run --unit=$master 'mongo -quiet --eval "rs.status().ok"')
echo "*** $SERVICE: Checking rs_ok($rs_ok) == 1 ..."
if [ "$rs_ok" -eq 1 ];then
    echo "=> Ok."
else
    echo "Trying to fix $SERVICE cluster:" 
    echo "temporary setting a resolvable hostname and restarting mongod"
    (set -xe
    ip=$(juju run --unit=$master 'unit-get private-address')
    juju run --unit=$master 'mongo -quiet --eval "printjson(rs.initiate({\"_id\": \"'$replset'\", \"members\": [ {\"_id\": 0, \"host\": \"'$ip':27017\" }] }))"'
    ) || exit 1
fi
typeset -i retries=10
until [ $retries -eq 0 ];do
    n_replicas=$(juju run --unit=$master 'mongo -quiet --eval "rs.status().members.length"')
    echo "*** $SERVICE: Checking n_replicas($n_replicas) == n_total($n_total) ; $retries retries left ..."
    if [ "$n_replicas" -eq "$n_total" ];then
        echo "=> Ok."
        break
    else
        echo "Need to add replicas ..."
        (set -x
        juju run --unit=$master 'mongo -quiet --eval "cfg=rs.conf();cfg.members[0].host=\"$(unit-get private-address):27017\";printjson(rs.reconfig(cfg))"'
        )
        for replica in $*;do
            ip=$(juju run --unit=$replica 'unit-get private-address')
            (set -x
            juju run --unit=$master 'mongo -quiet --eval "rs.add(\"'$ip':27017\");"'
            )
        done
    fi
    retries=retries-1
    sleep 10
done
# exit ok if there were retries left
[ $retries -gt 0 ]

E.g. run output

$ ./fix-mongo-cluster.sh
*** mongodb: checking clustered units...
units: mongodb/0 mongodb/1 mongodb/2
*** mongodb: Checking rs_ok(0) == 1 ...
Trying to fix mongodb cluster:
temporary setting a resolvable hostname and restarting mongod
++ juju run --unit=mongodb/0 'unit-get private-address'
+ ip=172.20.173.83
+ juju run --unit=mongodb/0 'mongo -quiet --eval "printjson(rs.initiate({\"_id\": \"myset\", \"members\": [ {\"_id\": 0, \"host\": \"172.20.173.83:27017\" }] }))"'
{
        "info" : "Config now saved locally.  Should come online in about a minute.",
        "ok" : 1
}
ERROR subprocess encountered error code 252
*** mongodb: Checking n_replicas(Mon Sep 29 14:11:24.491 TypeError: Cannot read property 'length' of undefined) == n_total(3) ; 10 retries left ...
./fix-mongo-cluster.sh: line 33: [: Mon Sep 29 14:11:24.491 TypeError: Cannot read property 'length' of undefined: integer expression expected
Need to add replicas ...
+ juju run --unit=mongodb/0 'mongo -quiet --eval "cfg=rs.conf();cfg.members[0].host=\"$(unit-get private-address):27017\";printjson(rs.reconfig(cfg))"'
{
        "startupStatus" : 6,
        "ok" : 0,
        "errmsg" : "Received replSetInitiate - should come online shortly."
}
Warning: Permanently added '172.20.173.103' (ECDSA) to the list of known hosts.
+ juju run --unit=mongodb/0 'mongo -quiet --eval "rs.add(\"172.20.173.103:27017\");"'
[object Object]
Warning: Permanently added '172.20.173.104' (ECDSA) to the list of known hosts.
+ juju run --unit=mongodb/0 'mongo -quiet --eval "rs.add(\"172.20.173.104:27017\");"'
[object Object]
*** mongodb: Checking n_replicas(2) == n_total(3) ; 9 retries left ...
Need to add replicas ...
+ juju run --unit=mongodb/0 'mongo -quiet --eval "cfg=rs.conf();cfg.members[0].host=\"$(unit-get private-address):27017\";printjson(rs.reconfig(cfg))"'
{ "ok" : 1 }
+ juju run --unit=mongodb/0 'mongo -quiet --eval "rs.add(\"172.20.173.103:27017\");"'
[object Object]
+ juju run --unit=mongodb/0 'mongo -quiet --eval "rs.add(\"172.20.173.104:27017\");"'
[object Object]
*** mongodb: Checking n_replicas(3) == n_total(3) ; 8 retries left ...
=> Ok.