rodrigogansobarbieri/recover_machine_agents.py

## recover_machine_agents.py
#!/usr/bin/env python3
"""
This is a tool for recovering lost machine units in current model

Usage:
{0} model-name dest-dir
"""

# changes summary:
# added logic to prevent overwrite /var/lib/juju to avoid removing non-machine units
# added logic for detecting path of mongo binary (e.g. if present in $PATH or is in /usr/lib/juju/mongo*/bin)
# added logic to determine primary mongo node to run queries against PRIMARY mongo
# added logic to also restore systemd unit files
# removed need to specify controller ip, as this is set when mongo PRIMARY detected

import json
import os
import shlex
import shutil
import subprocess
import sys
import tempfile

MONGOPASS_CMD = "juju ssh ubuntu@%s \"sudo grep ^apipassword: /var/lib/juju/agents/machine*/agent.conf\" | awk -e '{print $2}'"

MONGO_CMD = ""
MONGO_TEMPLATE = "juju ssh -m controller ubuntu@%s \"sudo %s --port 37017 --sslAllowInvalidCertificates --ssl --authenticationDatabase admin -u machine-%s -p %s juju < /home/ubuntu/%s 2>/dev/null\" | tail -n 2 | head -n 1"


def run(cmd, output=True, shell=True):
    print(cmd)
    if output:
        return subprocess.check_output(cmd, shell=shell).decode().strip()
    return subprocess.call(cmd, stderr=subprocess.PIPE,
        stdout=subprocess.PIPE, shell=shell)

def update_machine_password(controller, model, machine_number, passwordhash):
    file_content = """use juju
    db.machines.update({"model-uuid": "%s", "machineid": "%s"}, {$set:{"passwordhash": "%s"}})
    """ % (model, machine_number, passwordhash)

    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
        tmp_file.write(file_content)

    run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
    password = run(MONGOPASS_CMD % controller)
    run(MONGO_CMD % (controller, password, os.path.basename(tmp_file.name)))


def get_model_uuid(controller, model):
    file_content = """use juju
    db.models.find({"name": "%s"}, {"modeluuid": 1})
    """ % (model)

    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
        tmp_file.write(file_content)

    run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
    password = run(MONGOPASS_CMD % controller)
    uuid_json = run(MONGO_CMD % (controller, password, os.path.basename(
        tmp_file.name)))
    uuid = json.loads(uuid_json)['_id']
    return uuid

def determine_primary_mongo():
    global MONGO_CMD
    file_content = """use juju
    rs.isMaster()['primary']
    """

    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
        tmp_file.write(file_content)

    first_controller_num = run('juju machines -m controller |grep started|head -n1|cut -d " " -f1')
    mongo_cmd_tuple = gen_mongo_cmd(first_controller_num)
    temp_mongo_cmd = mongo_cmd_tuple[0] % os.path.basename(tmp_file.name)
    run("juju scp -m controller {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, first_controller_num))
    primary_ip = run(temp_mongo_cmd).split(':')[0]

    # now that primary established, we'll just use old logic for MONGO_CMD variable/pass
    # but hardcoding the machine number

    primary_controller_num = run('juju machines -m controller |grep {}|head -n1|cut -d " " -f1'.format(primary_ip))

    MONGO_CMD = "juju ssh ubuntu@%s \"sudo {} --port 37017 --sslAllowInvalidCertificates " \
            "--ssl --authenticationDatabase admin -u machine-{} -p %s juju < " \
            "/home/ubuntu/%s 2>/dev/null\" | tail -n 2 | head -n 1".format(mongo_cmd_tuple[1],primary_controller_num)

    return primary_ip


def gen_mongo_cmd(controller_num):
    binary_path = run('''juju ssh -m controller %s "ps aux|grep mongo"|grep -v grep|awk '{print $11}'|head -n1 | rev | cut -c 2- | rev''' % controller_num)
    #binary_path = "mongo"
    controller_pass =  run("juju ssh -m controller %s 'sudo grep ^apipassword: /var/lib/juju/agents/machine*/agent.conf' | awk -e '{print $2}'" % controller_num )
    return (MONGO_TEMPLATE % (controller_num, binary_path, controller_num, controller_pass, '%s'), binary_path)

def get_donor_password(controller, donor, model_uuid):
    file_content = """use juju
    db.machines.find({"model-uuid": "%s", "machineid": "%s"}, {"passwordhash": 1})
    """ % (model_uuid, donor)

    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
        tmp_file.write(file_content)

    run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
    password = run(MONGOPASS_CMD % controller)
    attributes_json = run(MONGO_CMD % (controller, password, os.path.basename(
        tmp_file.name)))
    attributes = json.loads(attributes_json)
    passwordhash = attributes['passwordhash']
    return passwordhash

def get_machine_nonce(controller, machine, model_uuid):
    file_content = """use juju
    db.machines.find({"model-uuid": "%s", "machineid": "%s"}, {"nonce": 1})
    """ % (model_uuid, machine)

    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
        tmp_file.write(file_content)

    print('get machine nonce reached...')
    run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
    password = run(MONGOPASS_CMD % controller)
    attributes_json = run(MONGO_CMD % (controller, password, os.path.basename(
        tmp_file.name)))
    attributes = json.loads(attributes_json)
    nonce = attributes['nonce']
    return nonce

def recover_machine(machine_number, juju_tar, donor, passwordhash, controller, model, systemd_tar):
    print("Recovering machine {}".format(machine_number))
    nonce = get_machine_nonce(controller, machine_number, model)
    check_juju_dir_or_create(machine_number)

    run("juju scp {} {}:/home/ubuntu/juju.tar".format(
        juju_tar, machine_number))
    run("juju ssh {} 'sudo tar -xvf /home/ubuntu/juju.tar -C /var/lib --skip-old-files --keep-directory-symlink --dereference'".format(machine_number))

    if 'lxd' in machine_number:
        machine_string = machine_number.replace('/', '-')
        run("juju ssh {} 'sudo mv -f /var/lib/juju/agents/machine-{} /var/lib/juju/agents/machine-{}'".format(
        machine_number, donor, machine_string))
        run("juju ssh {} 'sudo mv -f /var/lib/juju/tools/machine-{} /var/lib/juju/tools/machine-{}'".format(
        machine_number, donor, machine_string))
        run("juju ssh {} 'echo {} | sudo tee /var/lib/juju/nonce.txt'".format(machine_number, nonce))
    else:
        run("juju ssh {} 'sudo mv -f /var/lib/juju/agents/machine-{} /var/lib/juju/agents/machine-{}'".format(
        machine_number, donor, machine_number))
        run("juju ssh {} 'sudo mv -f /var/lib/juju/tools/machine-{} /var/lib/juju/tools/machine-{}'".format(
        machine_number, donor, machine_number))
        run("juju ssh {} 'echo {} | sudo tee /var/lib/juju/nonce.txt'".format(machine_number, nonce))

    # Update files

    if 'lxd' in machine_number:
        machine_string = machine_number.replace('/', '-')

        agent_file = "/var/lib/juju/agents/machine-{}/agent.conf".format(machine_string)

        run("juju ssh {} 'sudo sed -i \"s|tag: machine-{}|tag: machine-{}|g\" {}'".format(
            machine_number, donor, machine_string, agent_file))
        run("juju ssh {} 'sudo sed -i \"s|jujud-machine-{}|jujud-machine-{}|g\" {}'".format(
            machine_number, donor, machine_string, agent_file))
        run("juju ssh {} 'sudo sed -i \"s/nonce: .*/nonce: {}/g\" {}'".format(
            machine_number, nonce, agent_file))

    else:

        agent_file = "/var/lib/juju/agents/machine-{}/agent.conf".format(machine_number)

        run("juju ssh {} 'sudo sed -i \"s/tag: machine-{}/tag: machine-{}/g\" {}'".format(
            machine_number, donor, machine_number, agent_file))
        run("juju ssh {} 'sudo sed -i \"s/jujud-machine-{}/jujud-machine-{}/g\" {}'".format(
            machine_number, donor, machine_number, agent_file))
        run("juju ssh {} 'sudo sed -i \"s/nonce: .*/nonce: {}/g\" {}'".format(
            machine_number, nonce, agent_file))

    # restore unit files
    if 'lxd' in machine_number:
        run("juju scp {} {}:/home/ubuntu/systemd.tar".format(
            systemd_tar, machine_number))

        machine_string = machine_number.replace('/', '-')
        run("juju ssh {} 'sudo rm -rf /lib/systemd/system/jujud-machine-{}'".format(machine_number,machine_string))

        run("juju ssh {} 'sudo tar -xvf /home/ubuntu/systemd.tar -C / --skip-old-files'".format(machine_number))
        run("juju ssh {} 'sudo cp -nrp /lib/systemd/system/jujud-machine-{} /lib/systemd/system/jujud-machine-{}'".format(machine_number,donor,machine_string))
        run("juju ssh {} 'sudo cp -p /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service'".format(
                machine_number,donor, donor, machine_string, machine_string))

        exec_start_file = "/lib/systemd/system/jujud-machine-{}/exec-start.sh".format(machine_string)
        jujud_unit_file = "/lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_string, machine_string)

        run("juju ssh {} 'sudo sed -i \"s|machine-{}|machine-{}|g\" {}'".format(
            machine_number, donor, machine_string, exec_start_file))

        run("juju ssh {} 'sudo sed -i \"s|--machine-id {}|--machine-id {}|g\" {}'".format(
            machine_number, donor, machine_number, exec_start_file))

        run("juju ssh {} 'sudo sed -i \"s|machine-{}|machine-{}|g\" {}'".format(
            machine_number, donor, machine_string, jujud_unit_file))
    else:
        run("juju scp {} {}:/home/ubuntu/systemd.tar".format(
            systemd_tar, machine_number))
        run("juju ssh {} 'sudo rm -rf /lib/systemd/system/jujud-machine-{}'".format(machine_number,machine_number))

        run("juju ssh {} 'sudo tar -xvf /home/ubuntu/systemd.tar -C / --skip-old-files'".format(machine_number))

        run("juju ssh {} 'sudo cp -nrp /lib/systemd/system/jujud-machine-{} /lib/systemd/system/jujud-machine-{}'".format(machine_number,donor,machine_number))
        run("juju ssh {} 'sudo cp -p /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service'".format(
                machine_number,donor, donor, machine_number, machine_number))

        exec_start_file = "/lib/systemd/system/jujud-machine-{}/exec-start.sh".format(machine_number)
        jujud_unit_file = "/lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_number, machine_number)

        run("juju ssh {} 'sudo sed -i \"s/machine-{}/machine-{}/g\" {}'".format(
            machine_number, donor, machine_number, exec_start_file))

        run("juju ssh {} 'sudo sed -i \"s/--machine-id {}/--machine-id {}/g\" {}'".format(
            machine_number, donor, machine_number, exec_start_file))

        run("juju ssh {} 'sudo sed -i \"s/machine-{}/machine-{}/g\" {}'".format(
            machine_number, donor, machine_number, jujud_unit_file))


    # Update mongo
    update_machine_password(controller, model, machine_number, passwordhash)
    # Restart services
    link_to = determine_juju_version(machine_number)
    run("juju ssh {} 'for  u in $(sudo ls /var/lib/juju/agents/|sort); do sudo ln -sf /var/lib/juju/tools/{} /var/lib/juju/tools/$u; done'".format(machine_number,link_to))
    run("juju ssh {} sudo systemctl daemon-reload".format(machine_number))

    if 'lxd' in machine_number:
        machine_string = machine_number.replace('/', '-')
        run("juju ssh {} sudo systemctl enable /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_number,
            machine_string, machine_string))
        run("juju ssh {} sudo systemctl restart jujud-machine-{}".format(machine_number, machine_string))
    else:
        run("juju ssh {} sudo systemctl enable /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_number,
            machine_number, machine_number))
        run("juju ssh {} sudo systemctl restart jujud-machine-{}".format(machine_number, machine_number))


def check_juju_dir_or_create(machine_num):
        command =  "juju ssh ubuntu@{} 'sudo test -d /var/lib/juju'".format(machine_num)
        exit_code = run(command, output=False)
        if exit_code != 0:
            create_command = "juju ssh {} 'sudo mkdir /var/lib/juju'".format(machine_num)
            run(create_command)

        # check for existence of /var/lib/juju/agents/machine-$
        # remove it to avoid mv directory not-empty when recovering
        else:
            if 'lxd' in machine_num:
                machine_string = machine_num.replace('/', '-')
                check_for_machine_conf = "juju ssh ubuntu@{} 'sudo test -d /var/lib/juju/agents/machine-{}'".format(machine_num,machine_string)
            else:
                check_for_machine_conf = "juju ssh ubuntu@{} 'sudo test -d /var/lib/juju/agents/machine-{}'".format(machine_num,machine_num)

            if run(check_for_machine_conf, output=False) == 0:
                if 'lxd' in machine_num:
                     mv_existing_command = "juju ssh ubuntu@{} 'sudo rm -rf /var/lib/juju/agents/machine-{}'".format(machine_num,machine_num.replace('/', '-'))
                     run(mv_existing_command, output=False)
                else:
                     mv_existing_command = "juju ssh ubuntu@{} 'sudo rm -rf /var/lib/juju/agents/machine-{}'".format(machine_num,machine_num)
                     run(mv_existing_command, output=False)

def get_agent_from_donor(donor, destdir):
    print ("Getting agent from machine {}".format(donor))

    # Get dir from donor and remove everything except machine agent
    run("juju ssh {} 'sudo chmod -R a+r /var/lib/juju'".format(donor))
    run("juju ssh {} 'sudo tar -C /var/lib/ -cvf /tmp/juju.tar juju'".format(
        donor))
    run("juju scp {}:/tmp/juju.tar {}".format(donor, destdir))
    run("tar -xvf {}/juju.tar -C {}".format(destdir, destdir))
    run("rm -rf {}/juju/agents/unit*".format(destdir))
    run("rm -rf {}/juju/meter-status.yaml".format(destdir))
    run("rm -rf {}/juju/locks/*".format(destdir))
    run("rm -rf {}/juju/tools/unit*".format(destdir))
    run("rm -rf {}/juju/metricspool".format(destdir))
    run("rm -rf {}/juju/nonce.txt".format(destdir))
    run("rm -rf {}/juju.tar".format(destdir))
    run("tar -cvf {}/juju.tar -C {} juju".format(destdir, destdir))
    run("rm -rf {}/juju".format(destdir))
    get_donor_systemd_units(donor,destdir)


def get_donor_systemd_units(donor, destdir):
    run("juju ssh {} 'sudo tar -cvf /tmp/systemd.tar /lib/systemd/system/jujud-machine-{}'".format(donor,donor))
    run("juju scp {}:/tmp/systemd.tar {}".format(donor, destdir))


def determine_juju_version(machine_number):
    return run("juju ssh {} 'ls /var/lib/juju/tools |egrep ^2.|sort -n|head -n1'".format(machine_number), output=True)


def parse_machines_to_recover(status_json):
    machines = []
    units = []
    for app_name, app_data in status_json['applications'].items():
        for unit_name,unit_data in app_data['units'].items():
            if unit_data['juju-status']['current'] == "lost":
                units.append(unit_name)
                machine = unit_data['machine']
                if machine not in machines:
                    machines.append(machine)
    for machine_number, machine_data in status_json['machines'].items():
        if machine_data['juju-status']['current'] == "down":
            if machine_number not in machines:
                machines.append(machine_number)
    return machines, units


def parse_donor(status_json):
    candidates = []
    for machine_number, machine_data in status_json['machines'].items():
        if machine_data['juju-status']['current'] == "started":
            candidates.append(machine_number)
    for app_name, app_data in status_json['applications'].items():
        for unit_name,unit_data in app_data['units'].items():
            if unit_data['juju-status']['current'] == "lost":
                machine = unit_data['machine']
                if machine in candidates:
                    candidates.remove(machine)
    if len(candidates) > 0:
        return candidates[0]
    else:
        raise Exception("No donor candidates found")


def disable_units(units):
    for unit in units:
        service = unit.replace('/','-')
        output = run("juju ssh {} 'sudo systemctl -a | grep {}' || true".format(unit, service))
        if output:
            if 'active' in output:
                run("juju ssh {} 'sudo systemctl stop jujud-unit-{}.service'".format(unit, service))
            run("juju ssh {} 'sudo systemctl disable jujud-unit-{}.service'".format(unit, service))


def main():
    model = sys.argv[1]
    destdir = sys.argv[2]
    print("Determining the Primary MongoDB unit")
    controller = determine_primary_mongo()

    if os.path.exists(destdir):
        shutil.rmtree(destdir)
    os.mkdir(destdir)

    run("juju switch {}".format(model))

    model_uuid = get_model_uuid(controller, model)
    print(model_uuid)

    # Query first healthy unit to select as donor
    print("Attempting to find a healthy machine donor")
    status_string_json = run("juju status --format json")
    status_json = json.loads(status_string_json)
    donor = parse_donor(status_json)
    passwordhash = get_donor_password(controller, donor, model_uuid)
    print(passwordhash)
    get_agent_from_donor(donor, destdir)

    # slight modification to ensure it's 'Running'
    print("Getting list of machines to recover")
    status_string_json = run("juju status --format json")
    status_json = json.loads(status_string_json)
    machines, units = parse_machines_to_recover(status_json)
    disable_units(units)
    print("Machines to be recovered: {}".format(machines))
    for machine in machines:
        # added systemd_tar
        systemd_tar = destdir + "/systemd.tar"
        recover_machine(machine, "{}/juju.tar".format(destdir), donor, passwordhash, controller, model_uuid, systemd_tar)


def test():
    model = sys.argv[1]
    controller = sys.argv[2]
    destdir = sys.argv[3]

    if os.path.exists(destdir):
        shutil.rmtree(destdir)
    os.mkdir(destdir)

    run("juju switch {}".format(model))
    print(determine_primary_mongo())


if __name__ == "__main__":
    if len(sys.argv) == 1 :
        print(__doc__.format(sys.argv[0]))
        sys.exit(-1)

    main()
	#!/usr/bin/env python3
	"""
	This is a tool for recovering lost machine units in current model

	Usage:
	{0} model-name dest-dir
	"""

	# changes summary:
	# added logic to prevent overwrite /var/lib/juju to avoid removing non-machine units
	# added logic for detecting path of mongo binary (e.g. if present in $PATH or is in /usr/lib/juju/mongo*/bin)
	# added logic to determine primary mongo node to run queries against PRIMARY mongo
	# added logic to also restore systemd unit files
	# removed need to specify controller ip, as this is set when mongo PRIMARY detected

	import json
	import os
	import shlex
	import shutil
	import subprocess
	import sys
	import tempfile

	MONGOPASS_CMD = "juju ssh ubuntu@%s \"sudo grep ^apipassword: /var/lib/juju/agents/machine*/agent.conf\" \| awk -e '{print $2}'"

	MONGO_CMD = ""
	MONGO_TEMPLATE = "juju ssh -m controller ubuntu@%s \"sudo %s --port 37017 --sslAllowInvalidCertificates --ssl --authenticationDatabase admin -u machine-%s -p %s juju < /home/ubuntu/%s 2>/dev/null\" \| tail -n 2 \| head -n 1"


	def run(cmd, output=True, shell=True):
	print(cmd)
	if output:
	return subprocess.check_output(cmd, shell=shell).decode().strip()
	return subprocess.call(cmd, stderr=subprocess.PIPE,
	stdout=subprocess.PIPE, shell=shell)

	def update_machine_password(controller, model, machine_number, passwordhash):
	file_content = """use juju
	db.machines.update({"model-uuid": "%s", "machineid": "%s"}, {$set:{"passwordhash": "%s"}})
	""" % (model, machine_number, passwordhash)

	with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
	tmp_file.write(file_content)

	run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
	password = run(MONGOPASS_CMD % controller)
	run(MONGO_CMD % (controller, password, os.path.basename(tmp_file.name)))


	def get_model_uuid(controller, model):
	file_content = """use juju
	db.models.find({"name": "%s"}, {"modeluuid": 1})
	""" % (model)

	with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
	tmp_file.write(file_content)

	run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
	password = run(MONGOPASS_CMD % controller)
	uuid_json = run(MONGO_CMD % (controller, password, os.path.basename(
	tmp_file.name)))
	uuid = json.loads(uuid_json)['_id']
	return uuid

	def determine_primary_mongo():
	global MONGO_CMD
	file_content = """use juju
	rs.isMaster()['primary']
	"""

	with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
	tmp_file.write(file_content)

	first_controller_num = run('juju machines -m controller \|grep started\|head -n1\|cut -d " " -f1')
	mongo_cmd_tuple = gen_mongo_cmd(first_controller_num)
	temp_mongo_cmd = mongo_cmd_tuple[0] % os.path.basename(tmp_file.name)
	run("juju scp -m controller {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, first_controller_num))
	primary_ip = run(temp_mongo_cmd).split(':')[0]

	# now that primary established, we'll just use old logic for MONGO_CMD variable/pass
	# but hardcoding the machine number

	primary_controller_num = run('juju machines -m controller \|grep {}\|head -n1\|cut -d " " -f1'.format(primary_ip))

	MONGO_CMD = "juju ssh ubuntu@%s \"sudo {} --port 37017 --sslAllowInvalidCertificates " \
	"--ssl --authenticationDatabase admin -u machine-{} -p %s juju < " \
	"/home/ubuntu/%s 2>/dev/null\" \| tail -n 2 \| head -n 1".format(mongo_cmd_tuple[1],primary_controller_num)

	return primary_ip



	def gen_mongo_cmd(controller_num):
	binary_path = run('''juju ssh -m controller %s "ps aux\|grep mongo"\|grep -v grep\|awk '{print $11}'\|head -n1 \| rev \| cut -c 2- \| rev''' % controller_num)
	#binary_path = "mongo"
	controller_pass = run("juju ssh -m controller %s 'sudo grep ^apipassword: /var/lib/juju/agents/machine*/agent.conf' \| awk -e '{print $2}'" % controller_num )
	return (MONGO_TEMPLATE % (controller_num, binary_path, controller_num, controller_pass, '%s'), binary_path)

	def get_donor_password(controller, donor, model_uuid):
	file_content = """use juju
	db.machines.find({"model-uuid": "%s", "machineid": "%s"}, {"passwordhash": 1})
	""" % (model_uuid, donor)

	with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
	tmp_file.write(file_content)

	run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
	password = run(MONGOPASS_CMD % controller)
	attributes_json = run(MONGO_CMD % (controller, password, os.path.basename(
	tmp_file.name)))
	attributes = json.loads(attributes_json)
	passwordhash = attributes['passwordhash']
	return passwordhash

	def get_machine_nonce(controller, machine, model_uuid):
	file_content = """use juju
	db.machines.find({"model-uuid": "%s", "machineid": "%s"}, {"nonce": 1})
	""" % (model_uuid, machine)

	with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
	tmp_file.write(file_content)

	print('get machine nonce reached...')
	run("juju scp {} ubuntu@{}:/home/ubuntu".format(tmp_file.name, controller))
	password = run(MONGOPASS_CMD % controller)
	attributes_json = run(MONGO_CMD % (controller, password, os.path.basename(
	tmp_file.name)))
	attributes = json.loads(attributes_json)
	nonce = attributes['nonce']
	return nonce

	def recover_machine(machine_number, juju_tar, donor, passwordhash, controller, model, systemd_tar):
	print("Recovering machine {}".format(machine_number))
	nonce = get_machine_nonce(controller, machine_number, model)
	check_juju_dir_or_create(machine_number)

	run("juju scp {} {}:/home/ubuntu/juju.tar".format(
	juju_tar, machine_number))
	run("juju ssh {} 'sudo tar -xvf /home/ubuntu/juju.tar -C /var/lib --skip-old-files --keep-directory-symlink --dereference'".format(machine_number))

	if 'lxd' in machine_number:
	machine_string = machine_number.replace('/', '-')
	run("juju ssh {} 'sudo mv -f /var/lib/juju/agents/machine-{} /var/lib/juju/agents/machine-{}'".format(
	machine_number, donor, machine_string))
	run("juju ssh {} 'sudo mv -f /var/lib/juju/tools/machine-{} /var/lib/juju/tools/machine-{}'".format(
	machine_number, donor, machine_string))
	run("juju ssh {} 'echo {} \| sudo tee /var/lib/juju/nonce.txt'".format(machine_number, nonce))
	else:
	run("juju ssh {} 'sudo mv -f /var/lib/juju/agents/machine-{} /var/lib/juju/agents/machine-{}'".format(
	machine_number, donor, machine_number))
	run("juju ssh {} 'sudo mv -f /var/lib/juju/tools/machine-{} /var/lib/juju/tools/machine-{}'".format(
	machine_number, donor, machine_number))
	run("juju ssh {} 'echo {} \| sudo tee /var/lib/juju/nonce.txt'".format(machine_number, nonce))

	# Update files

	if 'lxd' in machine_number:
	machine_string = machine_number.replace('/', '-')

	agent_file = "/var/lib/juju/agents/machine-{}/agent.conf".format(machine_string)

	run("juju ssh {} 'sudo sed -i \"s\|tag: machine-{}\|tag: machine-{}\|g\" {}'".format(
	machine_number, donor, machine_string, agent_file))
	run("juju ssh {} 'sudo sed -i \"s\|jujud-machine-{}\|jujud-machine-{}\|g\" {}'".format(
	machine_number, donor, machine_string, agent_file))
	run("juju ssh {} 'sudo sed -i \"s/nonce: .*/nonce: {}/g\" {}'".format(
	machine_number, nonce, agent_file))

	else:

	agent_file = "/var/lib/juju/agents/machine-{}/agent.conf".format(machine_number)

	run("juju ssh {} 'sudo sed -i \"s/tag: machine-{}/tag: machine-{}/g\" {}'".format(
	machine_number, donor, machine_number, agent_file))
	run("juju ssh {} 'sudo sed -i \"s/jujud-machine-{}/jujud-machine-{}/g\" {}'".format(
	machine_number, donor, machine_number, agent_file))
	run("juju ssh {} 'sudo sed -i \"s/nonce: .*/nonce: {}/g\" {}'".format(
	machine_number, nonce, agent_file))

	# restore unit files
	if 'lxd' in machine_number:
	run("juju scp {} {}:/home/ubuntu/systemd.tar".format(
	systemd_tar, machine_number))

	machine_string = machine_number.replace('/', '-')
	run("juju ssh {} 'sudo rm -rf /lib/systemd/system/jujud-machine-{}'".format(machine_number,machine_string))

	run("juju ssh {} 'sudo tar -xvf /home/ubuntu/systemd.tar -C / --skip-old-files'".format(machine_number))
	run("juju ssh {} 'sudo cp -nrp /lib/systemd/system/jujud-machine-{} /lib/systemd/system/jujud-machine-{}'".format(machine_number,donor,machine_string))
	run("juju ssh {} 'sudo cp -p /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service'".format(
	machine_number,donor, donor, machine_string, machine_string))

	exec_start_file = "/lib/systemd/system/jujud-machine-{}/exec-start.sh".format(machine_string)
	jujud_unit_file = "/lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_string, machine_string)

	run("juju ssh {} 'sudo sed -i \"s\|machine-{}\|machine-{}\|g\" {}'".format(
	machine_number, donor, machine_string, exec_start_file))

	run("juju ssh {} 'sudo sed -i \"s\|--machine-id {}\|--machine-id {}\|g\" {}'".format(
	machine_number, donor, machine_number, exec_start_file))

	run("juju ssh {} 'sudo sed -i \"s\|machine-{}\|machine-{}\|g\" {}'".format(
	machine_number, donor, machine_string, jujud_unit_file))
	else:
	run("juju scp {} {}:/home/ubuntu/systemd.tar".format(
	systemd_tar, machine_number))
	run("juju ssh {} 'sudo rm -rf /lib/systemd/system/jujud-machine-{}'".format(machine_number,machine_number))

	run("juju ssh {} 'sudo tar -xvf /home/ubuntu/systemd.tar -C / --skip-old-files'".format(machine_number))

	run("juju ssh {} 'sudo cp -nrp /lib/systemd/system/jujud-machine-{} /lib/systemd/system/jujud-machine-{}'".format(machine_number,donor,machine_number))
	run("juju ssh {} 'sudo cp -p /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service'".format(
	machine_number,donor, donor, machine_number, machine_number))

	exec_start_file = "/lib/systemd/system/jujud-machine-{}/exec-start.sh".format(machine_number)
	jujud_unit_file = "/lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_number, machine_number)

	run("juju ssh {} 'sudo sed -i \"s/machine-{}/machine-{}/g\" {}'".format(
	machine_number, donor, machine_number, exec_start_file))

	run("juju ssh {} 'sudo sed -i \"s/--machine-id {}/--machine-id {}/g\" {}'".format(
	machine_number, donor, machine_number, exec_start_file))

	run("juju ssh {} 'sudo sed -i \"s/machine-{}/machine-{}/g\" {}'".format(
	machine_number, donor, machine_number, jujud_unit_file))


	# Update mongo
	update_machine_password(controller, model, machine_number, passwordhash)
	# Restart services
	link_to = determine_juju_version(machine_number)
	run("juju ssh {} 'for u in $(sudo ls /var/lib/juju/agents/\|sort); do sudo ln -sf /var/lib/juju/tools/{} /var/lib/juju/tools/$u; done'".format(machine_number,link_to))
	run("juju ssh {} sudo systemctl daemon-reload".format(machine_number))

	if 'lxd' in machine_number:
	machine_string = machine_number.replace('/', '-')
	run("juju ssh {} sudo systemctl enable /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_number,
	machine_string, machine_string))
	run("juju ssh {} sudo systemctl restart jujud-machine-{}".format(machine_number, machine_string))
	else:
	run("juju ssh {} sudo systemctl enable /lib/systemd/system/jujud-machine-{}/jujud-machine-{}.service".format(machine_number,
	machine_number, machine_number))
	run("juju ssh {} sudo systemctl restart jujud-machine-{}".format(machine_number, machine_number))


	def check_juju_dir_or_create(machine_num):
	command = "juju ssh ubuntu@{} 'sudo test -d /var/lib/juju'".format(machine_num)
	exit_code = run(command, output=False)
	if exit_code != 0:
	create_command = "juju ssh {} 'sudo mkdir /var/lib/juju'".format(machine_num)
	run(create_command)

	# check for existence of /var/lib/juju/agents/machine-$
	# remove it to avoid mv directory not-empty when recovering
	else:
	if 'lxd' in machine_num:
	machine_string = machine_num.replace('/', '-')
	check_for_machine_conf = "juju ssh ubuntu@{} 'sudo test -d /var/lib/juju/agents/machine-{}'".format(machine_num,machine_string)
	else:
	check_for_machine_conf = "juju ssh ubuntu@{} 'sudo test -d /var/lib/juju/agents/machine-{}'".format(machine_num,machine_num)

	if run(check_for_machine_conf, output=False) == 0:
	if 'lxd' in machine_num:
	mv_existing_command = "juju ssh ubuntu@{} 'sudo rm -rf /var/lib/juju/agents/machine-{}'".format(machine_num,machine_num.replace('/', '-'))
	run(mv_existing_command, output=False)
	else:
	mv_existing_command = "juju ssh ubuntu@{} 'sudo rm -rf /var/lib/juju/agents/machine-{}'".format(machine_num,machine_num)
	run(mv_existing_command, output=False)

	def get_agent_from_donor(donor, destdir):
	print ("Getting agent from machine {}".format(donor))

	# Get dir from donor and remove everything except machine agent
	run("juju ssh {} 'sudo chmod -R a+r /var/lib/juju'".format(donor))
	run("juju ssh {} 'sudo tar -C /var/lib/ -cvf /tmp/juju.tar juju'".format(
	donor))
	run("juju scp {}:/tmp/juju.tar {}".format(donor, destdir))
	run("tar -xvf {}/juju.tar -C {}".format(destdir, destdir))
	run("rm -rf {}/juju/agents/unit*".format(destdir))
	run("rm -rf {}/juju/meter-status.yaml".format(destdir))
	run("rm -rf {}/juju/locks/*".format(destdir))
	run("rm -rf {}/juju/tools/unit*".format(destdir))
	run("rm -rf {}/juju/metricspool".format(destdir))
	run("rm -rf {}/juju/nonce.txt".format(destdir))
	run("rm -rf {}/juju.tar".format(destdir))
	run("tar -cvf {}/juju.tar -C {} juju".format(destdir, destdir))
	run("rm -rf {}/juju".format(destdir))
	get_donor_systemd_units(donor,destdir)


	def get_donor_systemd_units(donor, destdir):
	run("juju ssh {} 'sudo tar -cvf /tmp/systemd.tar /lib/systemd/system/jujud-machine-{}'".format(donor,donor))
	run("juju scp {}:/tmp/systemd.tar {}".format(donor, destdir))


	def determine_juju_version(machine_number):
	return run("juju ssh {} 'ls /var/lib/juju/tools \|egrep ^2.\|sort -n\|head -n1'".format(machine_number), output=True)


	def parse_machines_to_recover(status_json):
	machines = []
	units = []
	for app_name, app_data in status_json['applications'].items():
	for unit_name,unit_data in app_data['units'].items():
	if unit_data['juju-status']['current'] == "lost":
	units.append(unit_name)
	machine = unit_data['machine']
	if machine not in machines:
	machines.append(machine)
	for machine_number, machine_data in status_json['machines'].items():
	if machine_data['juju-status']['current'] == "down":
	if machine_number not in machines:
	machines.append(machine_number)
	return machines, units


	def parse_donor(status_json):
	candidates = []
	for machine_number, machine_data in status_json['machines'].items():
	if machine_data['juju-status']['current'] == "started":
	candidates.append(machine_number)
	for app_name, app_data in status_json['applications'].items():
	for unit_name,unit_data in app_data['units'].items():
	if unit_data['juju-status']['current'] == "lost":
	machine = unit_data['machine']
	if machine in candidates:
	candidates.remove(machine)
	if len(candidates) > 0:
	return candidates[0]
	else:
	raise Exception("No donor candidates found")


	def disable_units(units):
	for unit in units:
	service = unit.replace('/','-')
	output = run("juju ssh {} 'sudo systemctl -a \| grep {}' \|\| true".format(unit, service))
	if output:
	if 'active' in output:
	run("juju ssh {} 'sudo systemctl stop jujud-unit-{}.service'".format(unit, service))
	run("juju ssh {} 'sudo systemctl disable jujud-unit-{}.service'".format(unit, service))


	def main():
	model = sys.argv[1]
	destdir = sys.argv[2]
	print("Determining the Primary MongoDB unit")
	controller = determine_primary_mongo()

	if os.path.exists(destdir):
	shutil.rmtree(destdir)
	os.mkdir(destdir)

	run("juju switch {}".format(model))

	model_uuid = get_model_uuid(controller, model)
	print(model_uuid)

	# Query first healthy unit to select as donor
	print("Attempting to find a healthy machine donor")
	status_string_json = run("juju status --format json")
	status_json = json.loads(status_string_json)
	donor = parse_donor(status_json)
	passwordhash = get_donor_password(controller, donor, model_uuid)
	print(passwordhash)
	get_agent_from_donor(donor, destdir)

	# slight modification to ensure it's 'Running'
	print("Getting list of machines to recover")
	status_string_json = run("juju status --format json")
	status_json = json.loads(status_string_json)
	machines, units = parse_machines_to_recover(status_json)
	disable_units(units)
	print("Machines to be recovered: {}".format(machines))
	for machine in machines:
	# added systemd_tar
	systemd_tar = destdir + "/systemd.tar"
	recover_machine(machine, "{}/juju.tar".format(destdir), donor, passwordhash, controller, model_uuid, systemd_tar)



	def test():
	model = sys.argv[1]
	controller = sys.argv[2]
	destdir = sys.argv[3]

	if os.path.exists(destdir):
	shutil.rmtree(destdir)
	os.mkdir(destdir)

	run("juju switch {}".format(model))
	print(determine_primary_mongo())



	if __name__ == "__main__":
	if len(sys.argv) == 1 :
	print(__doc__.format(sys.argv[0]))
	sys.exit(-1)

	main()