Lingnik/puppet_bootstrap_and_setup.py

## puppet_bootstrap_and_setup.py
#!/bin/env python
import time
import requests
import re

from infrastructure.models import Server
from utilities.models import ConnectionInfo

def run(job, logger=None, **kwargs):
    """
    This is a sample post-network verification action that can be used to boostrap
    a puppet agent onto the node being provisioned and do some initial setup.
    Namely, after bootstrapping the agent it adds the node to the group given in
    the action input, and runs the puppet agent to apply those changes.

    Before running this script, make sure to set up a ConnectionInfo with the name
    puppet_master_api_info that has the puppet master's FQDN in the IP/Hostname
    field, as well as the username and password for the API.

    You must provide the ID of the puppet master server in CB in the action input.
    That server must have credentials for remote scripts stored in CB.

    In addition, the server being provisioned must have credentials configured to
    allow CB to run remote scripts on it.
    """
    # first check the prerequisites: need to get puppet master CB server (by ID),
    # have credentials on that server for remote scripts, have the ConnectionInfo
    # with the puppet master's FQDN, username & password for API, and have credentials
    # for remote scripts on this server
    logger.info("Checking prerequisites")
    puppet_master_cb_id = {{ id_of_puppet_master_in_cb }}
    puppet_master_cb = Server.objects.filter(id=puppet_master_cb_id)
    if not puppet_master_cb:
        msg = ("The Puppet Master needs to be a server in CB. You said it has ID "
               "{}, but there is no such server.").format(puppet_master_cb_id)
        return "FAILURE", msg, ""
    puppet_master_cb = puppet_master_cb[0]

    puppet_master_cb_username = puppet_master_cb.get_credentials().get('username')
    puppet_master_cb_password = puppet_master_cb.get_credentials().get('password')
    if not puppet_master_cb_username or not puppet_master_cb_password:
        msg = ("The Puppet Master server in CB must have a username & password "
               "associated with it to use for running remote scripts.")
        return "FAILURE", msg, ""

    connection_info = ConnectionInfo.objects.filter(name='puppet_master_api_info')
    if not connection_info:
        msg = ("There must be a ConnectionInfo object with the name 'puppet_master"
               "_api_info' that contains the FQDN, API username & API password "
               "for the Puppet Master. No such object could be found.")
        return "FAILURE", msg, ""
    connection_info = connection_info[0]
    # FQDN is stored in IP field
    puppet_master_api_fqdn = connection_info.ip
    puppet_master_api_username = connection_info.username
    puppet_master_api_password = connection_info.password
    if not puppet_master_api_fqdn or not puppet_master_api_username or not puppet_master_api_password:
        msg = ("There is a ConnectionInfo object with the name 'puppet_master_api"
               "_info', but it is missing the necessary FQDN, API username, or "
               "API password.")
        return "FAILURE", msg, ""

    # prov jobs only ever have 1 server, so we'll take the first one
    server = job.server_set.first()
    assert isinstance(server, Server)
    server_username = server.get_credentials().get('username')
    server_password = server.get_credentials().get('password')
    if not server_username or not server_password:
        msg = ("The server being provisioned must have a username & password "
               "associated with it to use to running remote scripts.")
        return "FAILURE", msg, ""

    # BOOTSTRAP THE NODE'S AGENT & HANDLE CERTS
    # 1. Remote bootstrap the agent
    logger.info("Bootstrapping the agent")
    script = "curl -k https://{0}:8140/packages/current/install.bash | bash -s main:server={0}".format(
        puppet_master_api_fqdn)
    server.execute_script(script_contents=script, timeout=600)
    # Default behavior is to run agent as a daemon; stop & disable it
    server.execute_script(script_contents="service puppet stop")
    server.execute_script(script_contents="chkconfig puppet off")
    # It's OK for this to return abnormal exit codes, since its cert isn't signed yet
    try:
        server.execute_script(script_contents="/usr/local/bin/puppet agent -t")
    except RuntimeError:
        logger.debug("First call to puppet agent -t failed, but that's OK")
        pass

    # 2. Wait for Node to Request Certificate & Get Cert Name
    logger.info("Getting cert name")
    cert_name = ""
    while cert_name == "":
        cert_name = server.execute_script(script_contents="/usr/local/bin/facter networking.fqdn")
        time.sleep(5)

    # 3. Wait for Certificate Request to Appear on Puppet Master
    logger.info("Waiting for certificate request on Puppet Master")
    cert_req = ""
    num_retries = 20
    while cert_req == "" and num_retries > 0:
        # The grep may not succeed right away
        logger.debug("Will test for cert request {} more times".format(num_retries))
        try:
            # The cert list command only lists unsigned certs, so this will fail
            # if the node already has a signed cert
            cert_req = puppet_master_cb.execute_script(script_contents="/usr/local/bin/puppet cert list|grep \"{}\"".format(cert_name))
        except RuntimeError:
            if num_retries == 1:
                raise
            pass
        time.sleep(5)
        num_retries -= 1

    # (Synchronize servers with CloudBolt) ?????????? PENDING

    # Sign the Certificate
    logger.info("Signing the certificate")
    cert_sign = ""
    while cert_sign == "":
        cert_sign = puppet_master_cb.execute_script(script_contents="/usr/local/bin/puppet cert sign {}".format(cert_name))
        time.sleep(5)

    # ADD THE NODE TO THE GROUP
    logger.info("Adding the node to the group")
    # Get an RBAC API Token
    url = "https://{}:4433/rbac-api/v1/auth/token".format(puppet_master_api_fqdn)
    headers = {'Content-Type': 'application/json'}
    auth_data = {'login': puppet_master_api_username, 'password': puppet_master_api_password}
    r = requests.post(url, headers=headers, json=auth_data, verify=False)
    logger.debug("Response code for token request: {}".format(r.status_code))
    # Returns JSON that contains 'token' key, need value of that key
    pe_usertoken = r.json().get('token')

    # Get the Existing Group Object
    group = '{{ ID_of_group_in_Puppet }}'
    url = "https://{}:4433/classifier-api/v1/groups/{}".format(puppet_master_api_fqdn,
            group)
    headers = {'X-Authentication': pe_usertoken}
    r = requests.get(url, headers=headers, verify=False)
    logger.debug("Response code for getting group object: {}".format(r.status_code))
    group_details = r.json()
    logger.debug("Group details from API: {}".format(group_details))

    # Append the Node to the Group's Rules
    # The rule syntax is defined here: https://docs.puppetlabs.com/pe/latest/nc_groups.html#rule-condition-grammar
    # Assumption: noone else is messing with rules, so it always starts with ‘or’
    new_rule = ['=', 'name', '{}'.format(cert_name)]
    rules = group_details.get('rule')
    # If there are no rules, just set it to the new one; otherwise, append
    if rules:
        rules = rules + [new_rule]
    else:
        rules = ['or', new_rule]
    group_details['rule'] = rules

    # Assign Node to Requested Groups
    url = "https://{}:4433/classifier-api/v1/groups/{}".format(
            puppet_master_api_fqdn, group)
    headers = {'X-Authentication': pe_usertoken, 'Content-Type': 'application/json'}
    r = requests.post(url, headers=headers, json=group_details, verify=False)
    logger.debug("Response code for POSTing modified group: {}".format(r.status_code))

    # RUN PUPPET ON THE NODE
    logger.info("Running puppet on the new server")
    # Sync Node Until No More Work to Do
    agent_result = ""
    return_code = '2'
    # The puppet agent returns 2 if it's a success but there's more work to do,
    # 0 for success with no more work to do, and other for failure
    # But sometimes it gives an erroneous "bad" return code early on, so retry
    num_retries = 10
    p = re.compile(r'RETURNCODEFORCB=(?P<code>[0-9]*)')
    while return_code == '2' or (return_code not in ['0','2'] and num_retries > 0):
        agent_result = server.execute_script(script_contents="/usr/local/bin/puppet agent -t; echo RETURNCODEFORCB=$?")
        m = p.search(agent_result)
        return_code = m.group('code')
        logger.debug("Return code: {}".format(return_code))
        logger.debug("May retry up to {} more times if necessary".format(num_retries))
        time.sleep((15-num_retries)*5)
        num_retries -= 1
    if return_code != '0':
        return "FAILURE", "Error running puppet agent on node", "Result: {}".format(agent_result)

    return "", "", ""

## puppet_deprovision.sh
#!/bin/bash

NODE=10.60.60.161
PUPPETMASTER=pe-mono-2015-3.lab.iad.cloudboltsw.com
GROUP_HTTPD="487c3e29-004d-4ad4-b8af-3bc7f365ee3f"
GROUP_MYSQL="26f4a47b-e568-4d95-a67b-6a09f56bd52c"
GROUP_BACON="0229fdff-3790-415c-99a1-3f6fcc643fe9"
# GROUPS="$GROUP_HTTPD $GROUP_MYSQL $GROUP_BACON"
# This is for the API; This cannot be 'admin':
PE_USERNAME="puppet"
PE_PASSWORD="puppet"


# Wait for Node to Request Certificate & Get Cert Name
CERTNAME=""
while :
do
    CERTNAME=`ssh root@$NODE "facter networking.fqdn"`
    exitcode=$?
    if [ ! "$CERTNAME" == "" && $exitcode == 0 ]; then
        break
    fi
    # Wait five seconds
    sleep 5
done

# Stop the node's Puppet Agent
# If we don't do this, the node might check in while we're in the middle of getting rid of it.
ssh root@$NODE "service puppet stop"
ssh root@$NODE "chkconfig puppet off"


# Unpin the node from PE Groups
# -----------------------------

# Get an RBAC API Token
# There are a few ways we can authenticate going forward. One is to generate an API token given a PE username and password; others include whitelisting the server (which was the old way). The token method is pretty new.
PE_USERTOKEN=`curl -k -X POST -H 'Content-Type: application/json' -d "{\"login\": \"$PE_USERNAME\", \"password\": \"$PE_PASSWORD\"}" https://$PUPPETMASTER:4433/rbac-api/v1/auth/token|grep --color=never -Po '"'"token"'"\s*:\s*"\K([^"]*)'`

# Get the Existing Group Object
# The 'node' element on this object, if it exists, is the existing ruleset for the node-group mapping. If it doesn't exist, we need to create it.
GROUP_DETAILS=`curl -k -X GET https://$PUPPETMASTER:4433/classifier-api/v1/groups/$GROUP_BACON -H "X-Authentication:$PE_USERTOKEN"`

# Append the Node to the Group's Rules
# The rule syntax is defined here: https://docs.puppetlabs.com/pe/latest/nc_groups.html#rule-condition-grammar
# We can do a bit better job of parsing through this in multiline python code, e.g. when an 'or' isn't the first bool, etc. (unless CB is managing all of this from the get-go)
pycmd="import json; group=\"${GROUP_DETAILS//\"/\\\"}\"; jsoned=json.loads(group); rules=jsoned.get('rule', None); rules=None if not rules or len(rules)<=2 else [r for r in rules if not isinstance(r, list) or r[2]!=u'${CERTNAME}']; jsoned['rule']=rules; print json.dumps(jsoned);"
NEW_GROUP_DETAILS=`python -c "$pycmd"`

# POST Updated Group to PE
curl -k -X POST https://$PUPPETMASTER:4433/classifier-api/v1/groups/$GROUP_BACON -H "X-Authentication:$PE_USERTOKEN" -H "Content-Type: application/json" -d "$NEW_GROUP_DETAILS"
# The output should match $NEW_GROUP_DETAILS.


# Purge the node on the Puppetmaster
# ----------------------------------
# As of PE 2015.??? no longer need to restart memcached, reboot puppetmaster server, or make postgresql UPDATEs

# telling PM to remove from DB
ssh root@$PUPPETMASTER "puppet node purge $CERTNAME"

# Run puppet agent on PM to copy CRL (cert revocation list) to the CA SSL directory - finalize revoking cert
# this should loop until no changes (exit code 2 until 0)
ssh root@$PUPPETMASTER "puppet agent -t"

# Restart the Puppetmaster
# this ensures removal of the agent's cert from the cert list. If you don't do this, the node will check in again and re-register with PuppetDB, once again increasing the license count!
# You have to do this on any load-balanced masters in your system. - include this comment but don’t try to do
ssh root@$PUPPETMASTER "service pe-puppetserver restart"

# Proceed with Deprovisioning
# regular CB deprovisioning steps happen here

## puppet_provision.sh
#!/bin/bash

NODE=10.60.60.161
PUPPETMASTER=pe-mono-2015-3.lab.iad.cloudboltsw.com
GROUP_HTTPD="487c3e29-004d-4ad4-b8af-3bc7f365ee3f"
GROUP_MYSQL="26f4a47b-e568-4d95-a67b-6a09f56bd52c"
GROUP_BACON="0229fdff-3790-415c-99a1-3f6fcc643fe9"
# GROUPS="$GROUP_HTTPD $GROUP_MYSQL $GROUP_BACON"
# This is for the API; This cannot be 'admin':
PE_USERNAME="puppet"
PE_PASSWORD="puppet"

# Bootstrap Node
# --------------

# Remote Bootstrap Agent
# (from $src/cbhooks/hookmodules/puppet/pe-agent-bootstrap.py)
# see also: https://docs.puppetlabs.com/pe/latest/install_agents.html
ssh root@$NODE "curl -k https://$PUPPETMASTER:8140/packages/current/install.bash | bash -s main:server=$PUPPETMASTER"
# Options: run agent as daemon, run agent on cron, run agent on-demand
ssh root@$NODE "service puppet stop"
ssh root@$NODE "chkconfig puppet off"
# It's OK for this to return abnormal exit codes, since its cert isn't signed yet:
ssh root@$NODE "puppet agent -t" # || [[ $? == 2 ]]"

# Wait for Node to Request Certificate & Get Cert Name
CERTNAME=""
while :
do
    CERTNAME=`ssh root@$NODE "facter networking.fqdn"`
    exitcode=$?
    if [ ! "$CERTNAME" == "" && $exitcode == 0 ]; then
        break
    fi
    # Wait five seconds
    sleep 5
done

# Wait for Certificate Request to Appear on Puppetmaster
CERTREQ=""
while :
do
    CERTREQ=`ssh root@$PUPPETMASTER "puppet cert list|grep \"$CERTNAME\""`
    exitcode=$?
    if [ ! "$CERTREQ" == "" && $exitcode == 0 ]; then
        break
    fi
    sleep 5
done

# (Synchronize servers with CloudBolt)
# puppet_connector.sync_servers()

# Sign the Certificate
CERTSIGN=""
while :
do
    CERTSIGN=`ssh root@$PUPPETMASTER "puppet cert sign $CERTNAME"`
    exitcode=$?
    if [ ! "$CERTSIGN" == "" && $exitcode == 0 ]; then
        break
    fi
    sleep 5
done


# Add the Node to the Group
# -------------------------
# This can be automatic (based on a group hostname rule) or manual
# This is something that has changed numerous times: there used to be a `puppet node group`, then there was a Rake command… now we have to use the API! http://docs.puppetlabs.com/pe/2015.3/nc_forming_requests.html

# Get an RBAC API Token
# There are a few ways we can authenticate going forward. One is to generate an API token given a PE username and password; others include whitelisting the server (which was the old way). The token method is pretty new.
#Token lasts for 5 minutes
PE_USERTOKEN=`curl -k -X POST -H 'Content-Type: application/json' -d "{\"login\": \"$PE_USERNAME\", \"password\": \"$PE_PASSWORD\"}" https://$PUPPETMASTER:4433/rbac-api/v1/auth/token|grep --color=never -Po '"'"token"'"\s*:\s*"\K([^"]*)'`

# Get the Existing Group Object
# The 'node' element on this object, if it exists, is the existing ruleset for the node-group mapping. If it doesn't exist, we need to create it.
GROUP_DETAILS=`curl -k -X GET https://$PUPPETMASTER:4433/classifier-api/v1/groups/$GROUP_BACON -H "X-Authentication:$PE_USERTOKEN"`

# Append the Node to the Group's Rules
# The rule syntax is defined here: https://docs.puppetlabs.com/pe/latest/nc_groups.html#rule-condition-grammar
# We can do a bit better job of parsing through this in multiline python code, e.g. when an 'or' isn't the first bool, etc. (unless CB is managing all of this from the get-go)
# Assumption: noone else is messing with rules, so it always starts with ‘or’
pycmd="import json; group=\"${GROUP_DETAILS//\"/\\\"}\"; jsoned=json.loads(group); new_rule=[u'=', u'name', u'$CERTNAME']; rules=jsoned.get('rule'); rules=rules+[new_rule] if rules else [u'or', new_rule]; jsoned['rule']=rules; print json.dumps(jsoned);"
import json
jsoned = response.json()

new_rule=[u'=', u'name', u'$CERTNAME']
rules=jsoned.get('rule')
if rules:
	rules = rules + [new_rule]
else:
	rules = ['or', new_rule]
#rules=rules+[new_rule] if rules else [u'or', new_rule]
jsoned['rule']=rules
print json.dumps(jsoned);"
NEW_GROUP_DETAILS=`python -c "$pycmd"`

# ["or", ['~', 'name', 'hostname[0-9]*']]

# Assign Node to Requested Groups
curl -k -X POST https://$PUPPETMASTER:4433/classifier-api/v1/groups/$GROUP_BACON -H "X-Authentication:$PE_USERTOKEN" -H "Content-Type: application/json" -d "$NEW_GROUP_DETAILS"
# The output should match $NEW_GROUP_DETAILS.


# Run Puppet on the Node
# ----------------------

# Sync Node Until No More Work to Do
AGENT_RESULT=""
while :
do
    AGENT_RESULT=`ssh root@$NODE "puppet agent -t"`
#;echo RETURNCODE=$?"`   RETURNCODE=([0-9]*)
#AGENT_RESULT would have all output - parse return code out of last line (actually think it’s 1st, but should just be able to search) - 2=> try again, 0 => done, other => fail
    exitcode=$?
    echo "Puppet Agent Run Results (will run until no changes):"
    echo $AGENT_RESULT
    if [ ! "$AGENT_RESULT" == "" && $exitcode == 0 ]; then
        break
    fi
    sleep 5
done

# Check Puppet for Reports that demonstrate the groups are associated?

# Sync CB Server with Node

## remove_server_from_puppet.py
#!/bin/env python
import time
import requests
import re

from infrastructure.models import Server
from utilities.models import ConnectionInfo

def run(job, logger=None, **kwargs):
    """
    This is a sample post-network verification action that can be used to boostrap
    a puppet agent onto the node being provisioned and do some initial setup.
    Namely, after bootstrapping the agent it adds the node to the group given in
    the action input, and runs the puppet agent to apply those changes.

    Before running this script, make sure to set up a ConnectionInfo with the name
    puppet_master_api_info that has the puppet master's FQDN in the IP/Hostname
    field, as well as the username and password for the API.

    You must provide the ID of the puppet master server in CB in the action input.
    That server must have credentials for remote scripts stored in CB.

    In addition, the server being provisioned must have credentials configured to
    allow CB to run remote scripts on it.
    """
    # first check the prerequisites: need to get puppet master CB server (by ID),
    # have credentials on that server for remote scripts, have the ConnectionInfo
    # with the puppet master's FQDN, username & password for API, and have credentials
    # for remote scripts on this server
    logger.info("Checking prerequisites")
    puppet_master_cb_id = {{ id_of_puppet_master_in_cb }}
    puppet_master_cb = Server.objects.filter(id=puppet_master_cb_id)
    if not puppet_master_cb:
        msg = ("The Puppet Master needs to be a server in CB. You said it has ID "
               "{}, but there is no such server.").format(puppet_master_cb_id)
        return "FAILURE", msg, ""
    puppet_master_cb = puppet_master_cb[0]

    puppet_master_cb_username = puppet_master_cb.get_credentials().get('username')
    puppet_master_cb_password = puppet_master_cb.get_credentials().get('password')
    if not puppet_master_cb_username or not puppet_master_cb_password:
        msg = ("The Puppet Master server in CB must have a username & password "
               "associated with it to use for running remote scripts.")
        return "FAILURE", msg, ""

    connection_info = ConnectionInfo.objects.filter(name='puppet_master_api_info')
    if not connection_info:
        msg = ("There must be a ConnectionInfo object with the name 'puppet_master"
               "_api_info' that contains the FQDN, API username & API password "
               "for the Puppet Master. No such object could be found.")
        return "FAILURE", msg, ""
    connection_info = connection_info[0]
    # FQDN is stored in IP field
    puppet_master_api_fqdn = connection_info.ip
    puppet_master_api_username = connection_info.username
    puppet_master_api_password = connection_info.password
    if not puppet_master_api_fqdn or not puppet_master_api_username or not puppet_master_api_password:
        msg = ("There is a ConnectionInfo object with the name 'puppet_master_api"
               "_info', but it is missing the necessary FQDN, API username, or "
               "API password.")
        return "FAILURE", msg, ""

    server = job.server_set.first()
    assert isinstance(server, Server)
    server_username = server.get_credentials().get('username')
    server_password = server.get_credentials().get('password')
    if not server_username or not server_password:
        msg = ("The server being provisioned must have a username & password "
               "associated with it to use to running remote scripts.")
        return "FAILURE", msg, ""

    # Get Cert Name
    logger.info("Getting cert name")
    cert_name = ""
    while cert_name == "":
        cert_name = server.execute_script(script_contents="/usr/local/bin/facter networking.fqdn")
        time.sleep(5)

    # Stop the node's Puppet Agent
    # If we don't do this, the node might check in while we're in the middle of getting rid of it.
    server.execute_script(script_contents="service puppet stop")
    server.execute_script(script_contents="chkconfig puppet off")

    # UNPIN THE NODE FROM THE GIVEN GROUP
    logger.info("Unpinning the node from the group")
    # Get an RBAC API Token
    url = "https://{}:4433/rbac-api/v1/auth/token".format(puppet_master_api_fqdn)
    headers = {'Content-Type': 'application/json'}
    auth_data = {'login': puppet_master_api_username, 'password': puppet_master_api_password}
    r = requests.post(url, headers=headers, json=auth_data, verify=False)
    logger.debug("Response code for token request: {}".format(r.status_code))
    # Returns JSON that contains 'token' key, need value of that key
    pe_usertoken = r.json().get('token')

    # Get the Existing Group Object
    group = '{{ ID_of_group_in_Puppet }}'
    url = "https://{}:4433/classifier-api/v1/groups/{}".format(puppet_master_api_fqdn,
            group)
    headers = {'X-Authentication': pe_usertoken}
    r = requests.get(url, headers=headers, verify=False)
    logger.debug("Response code for getting group object: {}".format(r.status_code))
    group_details = r.json()
    logger.debug("Group details from API: {}".format(group_details))

    # Remove the Node from the Group's Rules
    # The rule syntax is defined here: https://docs.puppetlabs.com/pe/latest/nc_groups.html#rule-condition-grammar
    rules = group_details.get('rule')
    if rules:
        # The initial boolean is not a list; keep it
        # For the subsequent lists, only keep those where the 3rd item (value to
        # compare) is not the name of this node's cert
        rules = [r for r in rules if not isinstance(r, list) or r[2] != cert_name]
        # If after removing this node there's only one item, it's the initial
        # boolean operator only, so just set rules to None
        if len(rules) == 1:
            rules = None
    group_details['rule'] = rules

    # Update the Group in Puppet
    url = "https://{}:4433/classifier-api/v1/groups/{}".format(
            puppet_master_api_fqdn, group)
    headers = {'X-Authentication': pe_usertoken, 'Content-Type': 'application/json'}
    r = requests.post(url, headers=headers, json=group_details, verify=False)
    logger.debug("Response code for POSTing modified group: {}".format(r.status_code))

    # PURGE NODE ON PUPPET MASTER
    logger.info("Purging the node on the Puppet Master")
    puppet_master_cb.execute_script(script_contents="/usr/local/bin/puppet node purge {}".format(cert_name))
    # Run puppet agent on PM to copy CRL to the CA SSL directory AKA finalize revoking cert
    logger.info("Running agent on Puppet Master to finalize revoking cert")
    agent_result = ""
    return_code = '2'
    # The puppet agent returns 2 if it's a success but there's more work to do,
    # 0 for success with no more work to do, and other for failure
    # But sometimes it gives an erroneous "bad" return code early on, so retry
    num_retries = 10
    p = re.compile(r'RETURNCODEFORCB=(?P<code>[0-9]*)')
    while return_code == '2' or (return_code not in ['0','2'] and num_retries > 0):
        agent_result = puppet_master_cb.execute_script(script_contents="/usr/local/bin/puppet agent -t; echo RETURNCODEFORCB=$?")
        m = p.search(agent_result)
        return_code = m.group('code')
        logger.debug("Return code: {}".format(return_code))
        logger.debug("May retry up to {} more times if necessary".format(num_retries))
        time.sleep((15-num_retries)*5)
        num_retries -= 1
    if return_code != '0':
        return "FAILURE", "Error running puppet agent on Puppet Master", "Result: {}".format(agent_result)

    # Restart the Puppet Master
    # This ensures removal of the agent's cert from the cert list. If you don't do this, the node will check in again and re-register with PuppetDB, once again increasing the license count!
    # You have to do this on any load-balanced masters in your system.
    logger.info("Restarting Puppet Master to ensure removal of cert")
    puppet_master_cb.execute_script(script_contents="service pe-puppetserver restart")

    return "", "", ""
	#!/bin/env python
	import time
	import requests
	import re

	from infrastructure.models import Server
	from utilities.models import ConnectionInfo

	def run(job, logger=None, **kwargs):
	"""
	This is a sample post-network verification action that can be used to boostrap
	a puppet agent onto the node being provisioned and do some initial setup.
	Namely, after bootstrapping the agent it adds the node to the group given in
	the action input, and runs the puppet agent to apply those changes.

	Before running this script, make sure to set up a ConnectionInfo with the name
	puppet_master_api_info that has the puppet master's FQDN in the IP/Hostname
	field, as well as the username and password for the API.

	You must provide the ID of the puppet master server in CB in the action input.
	That server must have credentials for remote scripts stored in CB.

	In addition, the server being provisioned must have credentials configured to
	allow CB to run remote scripts on it.
	"""
	# first check the prerequisites: need to get puppet master CB server (by ID),
	# have credentials on that server for remote scripts, have the ConnectionInfo
	# with the puppet master's FQDN, username & password for API, and have credentials
	# for remote scripts on this server
	logger.info("Checking prerequisites")
	puppet_master_cb_id = {{ id_of_puppet_master_in_cb }}
	puppet_master_cb = Server.objects.filter(id=puppet_master_cb_id)
	if not puppet_master_cb:
	msg = ("The Puppet Master needs to be a server in CB. You said it has ID "
	"{}, but there is no such server.").format(puppet_master_cb_id)
	return "FAILURE", msg, ""
	puppet_master_cb = puppet_master_cb[0]

	puppet_master_cb_username = puppet_master_cb.get_credentials().get('username')
	puppet_master_cb_password = puppet_master_cb.get_credentials().get('password')
	if not puppet_master_cb_username or not puppet_master_cb_password:
	msg = ("The Puppet Master server in CB must have a username & password "
	"associated with it to use for running remote scripts.")
	return "FAILURE", msg, ""

	connection_info = ConnectionInfo.objects.filter(name='puppet_master_api_info')
	if not connection_info:
	msg = ("There must be a ConnectionInfo object with the name 'puppet_master"
	"_api_info' that contains the FQDN, API username & API password "
	"for the Puppet Master. No such object could be found.")
	return "FAILURE", msg, ""
	connection_info = connection_info[0]
	# FQDN is stored in IP field
	puppet_master_api_fqdn = connection_info.ip
	puppet_master_api_username = connection_info.username
	puppet_master_api_password = connection_info.password
	if not puppet_master_api_fqdn or not puppet_master_api_username or not puppet_master_api_password:
	msg = ("There is a ConnectionInfo object with the name 'puppet_master_api"
	"_info', but it is missing the necessary FQDN, API username, or "
	"API password.")
	return "FAILURE", msg, ""

	# prov jobs only ever have 1 server, so we'll take the first one
	server = job.server_set.first()
	assert isinstance(server, Server)
	server_username = server.get_credentials().get('username')
	server_password = server.get_credentials().get('password')
	if not server_username or not server_password:
	msg = ("The server being provisioned must have a username & password "
	"associated with it to use to running remote scripts.")
	return "FAILURE", msg, ""

	# BOOTSTRAP THE NODE'S AGENT & HANDLE CERTS
	# 1. Remote bootstrap the agent
	logger.info("Bootstrapping the agent")
	script = "curl -k https://{0}:8140/packages/current/install.bash \| bash -s main:server={0}".format(
	puppet_master_api_fqdn)
	server.execute_script(script_contents=script, timeout=600)
	# Default behavior is to run agent as a daemon; stop & disable it
	server.execute_script(script_contents="service puppet stop")
	server.execute_script(script_contents="chkconfig puppet off")
	# It's OK for this to return abnormal exit codes, since its cert isn't signed yet
	try:
	server.execute_script(script_contents="/usr/local/bin/puppet agent -t")
	except RuntimeError:
	logger.debug("First call to puppet agent -t failed, but that's OK")
	pass

	# 2. Wait for Node to Request Certificate & Get Cert Name
	logger.info("Getting cert name")
	cert_name = ""
	while cert_name == "":
	cert_name = server.execute_script(script_contents="/usr/local/bin/facter networking.fqdn")
	time.sleep(5)

	# 3. Wait for Certificate Request to Appear on Puppet Master
	logger.info("Waiting for certificate request on Puppet Master")
	cert_req = ""
	num_retries = 20
	while cert_req == "" and num_retries > 0:
	# The grep may not succeed right away
	logger.debug("Will test for cert request {} more times".format(num_retries))
	try:
	# The cert list command only lists unsigned certs, so this will fail
	# if the node already has a signed cert
	cert_req = puppet_master_cb.execute_script(script_contents="/usr/local/bin/puppet cert list\|grep \"{}\"".format(cert_name))
	except RuntimeError:
	if num_retries == 1:
	raise
	pass
	time.sleep(5)
	num_retries -= 1

	# (Synchronize servers with CloudBolt) ?????????? PENDING

	# Sign the Certificate
	logger.info("Signing the certificate")
	cert_sign = ""
	while cert_sign == "":
	cert_sign = puppet_master_cb.execute_script(script_contents="/usr/local/bin/puppet cert sign {}".format(cert_name))
	time.sleep(5)

	# ADD THE NODE TO THE GROUP
	logger.info("Adding the node to the group")
	# Get an RBAC API Token
	url = "https://{}:4433/rbac-api/v1/auth/token".format(puppet_master_api_fqdn)
	headers = {'Content-Type': 'application/json'}
	auth_data = {'login': puppet_master_api_username, 'password': puppet_master_api_password}
	r = requests.post(url, headers=headers, json=auth_data, verify=False)
	logger.debug("Response code for token request: {}".format(r.status_code))
	# Returns JSON that contains 'token' key, need value of that key
	pe_usertoken = r.json().get('token')

	# Get the Existing Group Object
	group = '{{ ID_of_group_in_Puppet }}'
	url = "https://{}:4433/classifier-api/v1/groups/{}".format(puppet_master_api_fqdn,
	group)
	headers = {'X-Authentication': pe_usertoken}
	r = requests.get(url, headers=headers, verify=False)
	logger.debug("Response code for getting group object: {}".format(r.status_code))
	group_details = r.json()
	logger.debug("Group details from API: {}".format(group_details))

	# Append the Node to the Group's Rules
	# The rule syntax is defined here: https://docs.puppetlabs.com/pe/latest/nc_groups.html#rule-condition-grammar
	# Assumption: noone else is messing with rules, so it always starts with ‘or’
	new_rule = ['=', 'name', '{}'.format(cert_name)]
	rules = group_details.get('rule')
	# If there are no rules, just set it to the new one; otherwise, append
	if rules:
	rules = rules + [new_rule]
	else:
	rules = ['or', new_rule]
	group_details['rule'] = rules

	# Assign Node to Requested Groups
	url = "https://{}:4433/classifier-api/v1/groups/{}".format(
	puppet_master_api_fqdn, group)
	headers = {'X-Authentication': pe_usertoken, 'Content-Type': 'application/json'}
	r = requests.post(url, headers=headers, json=group_details, verify=False)
	logger.debug("Response code for POSTing modified group: {}".format(r.status_code))

	# RUN PUPPET ON THE NODE
	logger.info("Running puppet on the new server")
	# Sync Node Until No More Work to Do
	agent_result = ""
	return_code = '2'
	# The puppet agent returns 2 if it's a success but there's more work to do,
	# 0 for success with no more work to do, and other for failure
	# But sometimes it gives an erroneous "bad" return code early on, so retry
	num_retries = 10
	p = re.compile(r'RETURNCODEFORCB=(?P<code>[0-9]*)')
	while return_code == '2' or (return_code not in ['0','2'] and num_retries > 0):
	agent_result = server.execute_script(script_contents="/usr/local/bin/puppet agent -t; echo RETURNCODEFORCB=$?")
	m = p.search(agent_result)
	return_code = m.group('code')
	logger.debug("Return code: {}".format(return_code))
	logger.debug("May retry up to {} more times if necessary".format(num_retries))
	time.sleep((15-num_retries)*5)
	num_retries -= 1
	if return_code != '0':
	return "FAILURE", "Error running puppet agent on node", "Result: {}".format(agent_result)

	return "", "", ""
	#!/bin/bash

	NODE=10.60.60.161
	PUPPETMASTER=pe-mono-2015-3.lab.iad.cloudboltsw.com
	GROUP_HTTPD="487c3e29-004d-4ad4-b8af-3bc7f365ee3f"
	GROUP_MYSQL="26f4a47b-e568-4d95-a67b-6a09f56bd52c"
	GROUP_BACON="0229fdff-3790-415c-99a1-3f6fcc643fe9"
	# GROUPS="$GROUP_HTTPD $GROUP_MYSQL $GROUP_BACON"
	# This is for the API; This cannot be 'admin':
	PE_USERNAME="puppet"
	PE_PASSWORD="puppet"


	# Wait for Node to Request Certificate & Get Cert Name
	CERTNAME=""
	while :
	do
	CERTNAME=`ssh root@$NODE "facter networking.fqdn"`
	exitcode=$?
	if [ ! "$CERTNAME" == "" && $exitcode == 0 ]; then
	break
	fi
	# Wait five seconds
	sleep 5
	done

	# Stop the node's Puppet Agent
	# If we don't do this, the node might check in while we're in the middle of getting rid of it.
	ssh root@$NODE "service puppet stop"
	ssh root@$NODE "chkconfig puppet off"


	# Unpin the node from PE Groups
	# -----------------------------

	# Get an RBAC API Token
	# There are a few ways we can authenticate going forward. One is to generate an API token given a PE username and password; others include whitelisting the server (which was the old way). The token method is pretty new.
	PE_USERTOKEN=`curl -k -X POST -H 'Content-Type: application/json' -d "{\"login\": \"$PE_USERNAME\", \"password\": \"$PE_PASSWORD\"}" https://$PUPPETMASTER:4433/rbac-api/v1/auth/token\|grep --color=never -Po '"'"token"'"\s:\s"\K([^"]*)'`

	# Get the Existing Group Object
	# The 'node' element on this object, if it exists, is the existing ruleset for the node-group mapping. If it doesn't exist, we need to create it.
	GROUP_DETAILS=`curl -k -X GET https://$PUPPETMASTER:4433/classifier-api/v1/groups/$GROUP_BACON -H "X-Authentication:$PE_USERTOKEN"`

	# Append the Node to the Group's Rules
	# The rule syntax is defined here: https://docs.puppetlabs.com/pe/latest/nc_groups.html#rule-condition-grammar
	# We can do a bit better job of parsing through this in multiline python code, e.g. when an 'or' isn't the first bool, etc. (unless CB is managing all of this from the get-go)
	pycmd="import json; group=\"${GROUP_DETAILS//\"/\\\"}\"; jsoned=json.loads(group); rules=jsoned.get('rule', None); rules=None if not rules or len(rules)<=2 else [r for r in rules if not isinstance(r, list) or r[2]!=u'${CERTNAME}']; jsoned['rule']=rules; print json.dumps(jsoned);"
	NEW_GROUP_DETAILS=`python -c "$pycmd"`

	# POST Updated Group to PE
	curl -k -X POST https://$PUPPETMASTER:4433/classifier-api/v1/groups/$GROUP_BACON -H "X-Authentication:$PE_USERTOKEN" -H "Content-Type: application/json" -d "$NEW_GROUP_DETAILS"
	# The output should match $NEW_GROUP_DETAILS.


	# Purge the node on the Puppetmaster
	# ----------------------------------
	# As of PE 2015.??? no longer need to restart memcached, reboot puppetmaster server, or make postgresql UPDATEs

	# telling PM to remove from DB
	ssh root@$PUPPETMASTER "puppet node purge $CERTNAME"

	# Run puppet agent on PM to copy CRL (cert revocation list) to the CA SSL directory - finalize revoking cert
	# this should loop until no changes (exit code 2 until 0)
	ssh root@$PUPPETMASTER "puppet agent -t"

	# Restart the Puppetmaster
	# this ensures removal of the agent's cert from the cert list. If you don't do this, the node will check in again and re-register with PuppetDB, once again increasing the license count!
	# You have to do this on any load-balanced masters in your system. - include this comment but don’t try to do
	ssh root@$PUPPETMASTER "service pe-puppetserver restart"

	# Proceed with Deprovisioning
	# regular CB deprovisioning steps happen here