mancdaz/rcpe_deploy-new

## gistfile1.txt
{
 "id": "rcb deployment setup",
 "description": "Setup for deploy script",
 "attributes": {
   "network": {
     "reserved": {
     	    "bastion": "172.31.0.5",
            "pxeapp": "172.31.0.6",
            "infra": "172.31.0.9",
            "infra_mac": "60:eb:69:6e:f0:31",
	    "infra_drac": "172.31.0.4",
            "cont_mac": "aa:bb:cc:dd:ee:ff",
            "crowbar": "172.31.0.10",
            "netmask": "255.255.255.0",
            "gateway": "172.31.0.5",
            "nameserver": "64.39.2.170",
            "cbfqdn": "lab.rcb.me",
            "nodecount": "14"
     }
   }
 }
}

## rcpe_deploy-new
#!/bin/bash

## TURN ON FOR DEBUGGING
# set -e
# set -x

## URL's for downloading the PXEAPPLIANCE
PXE_IMAGE_URL=${PXE_IMAGE_URL:-http://c271871.r71.cf1.rackcdn.com/pxeappliance_gold.qcow2}
PXE_XML_URL=${PXE_XML_URL:-http://c271871.r71.cf1.rackcdn.com/pxeappliance.xml}

# FOR CROWBAR PROPOSALS
PROPOSAL_NAME="openstack"

## Cleanup function for all events
function cleanup() {
    echo "Exiting..."
    virsh destroy pxeappliance
    virsh undefine pxeappliance
    qemu-nbd -d /dev/nbd0
    rm -rf /var/lock/shep_protection.lock
    rm -rf /opt/rcb/*
    exit $?
}

## SET INTERRUPT EXIT
trap cleanup SIGINT

## GRAB DRAC AND CROWBAR CREDENTIALS:
## NOTE: You must create a .creds file with DRAC USER and PASSWORD
## File should contain DUSERNAME, DPASSWORD, CUSERNAME, CPASSWORD
DIR="$( cd "$( dirname "$0" )" && pwd )"
if [ -f $DIR/.creds ]; then
    source .creds
else
    echo "Missing creds file."
    cleanup
    exit $?
fi

## Ensure i'm not already running (Shep protection)
if [ -f /var/lock/shep_protection.lock ]; then
    echo "Running already"
    exit 1
else
    touch /var/lock/shep_protection.lock
fi

## Simple function for ipmi boot of infra node
function ipmi_pxeboot() {
    POWERSTATE=`ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis status | grep System | awk '{print $4}'`
    if [ "$POWERSTATE" == 'on' ]; then
        for i in $(seq 1 5); do
            /usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis bootdev pxe
            /usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis power cycle
        done
    else
        for i in $(seq 1 5); do
           /usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis bootdev pxe
           /usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis power on
        done
    fi
}

## Given x minutes, a node, and a port test response for x minutes. If the port doesn't become available in said time
## exit and clean up.
function port_test() {
    # $1 - Minutes to wait
    # $2 - NODE to test
    # $3 - Port to test (netcat)
    maxwait=$1
    node=$2
    port=$3
    count=1
    while [ $count -lt $maxwait ]; do
        count=$(( count + 1 ))
        sleep 60s
        if ( nc $node $port -w 1 -q 0 < /dev/null ); then
            break
        fi
        if [ $count -eq $maxwait ]; then
            echo "Admin/Infra node is not network accessible"
            cleanup
            exit 1
        fi
    done
}

## Given a service name and a action create/commit a crowbar proposal. This in effect installs service software on node
## determined by crowbar discovery.
function crowbar_proposal() {
    # $1 - Service Name
    # $2 - Action (create|commit)
    service=$1
    action=$2
    cmd="/opt/dell/bin/crowbar_${service} -U ${CUSERNAME} -P ${CPASSWORD}"
    echo "Executing crowbar_proposal using:"
    echo " Service: ${service}"
    echo " Action: ${action}"

    if ! ( sudo -u rcb -- ssh ${SSH_OPTS} crowbar@${CROWBAR} "${cmd} proposal ${action} ${PROPOSAL_NAME}" ); then
        echo "Unable to ${action} the ${service} Proposal"
        cleanup
        exit 1
    fi
}

function crowbar_proposal_edit() {

    # $1 - Service name
    # $2 - second infra mac
    service=$1
    mac=$2
    cmd="/opt/dell/bin/crowbar_${service} -U ${CUSERNAME} -P ${CPASSWORD}"
    echo "Editing crowbar_proposal using:"
    echo " Service: ${service}"
    echo " Action: edit"

    # make the mac address into a crowbar friendly hostname
    target_host="$(echo d${mac}.${CBFQDN} | sed -e s/\:/\-/)"

    # urgh need to fix so it dumps out on remote node rather than locally
    sudo -u rcb -- ssh ${SSH_OPTS} crowbar@${CROWBAR} "${cmd} proposal show ${PROPOSAL_NAME} > /tmp/$service.json; \
    current_host=$(grep $CBFQDN  $service.json|sed -e 's/^[ \t]*//'|sed -e 's/\"//g' ) ; \
    sed -i -e s/$current_host/$target_host/ $service.json; \
    ${cmd} proposal edit $service --file=/tmp/$service.json"

}


## Given service name and time to wat check the status of the above proposal until Active.
function crowbar_proposal_status() {
    # $1 - Service Name
    # $2 - Wait Time
    service=$1
    wait_timer=${2:-15} # Default to 15 minutes if no wait_time provided
    cmd="/opt/dell/bin/crowbar_${service} -U ${CUSERNAME} -P ${CPASSWORD}"
    echo "Executing crowbar_proposal using:"
    echo " Service: ${service}"
    echo " Wait Time: ${wait_timer}"

    count=1
    while [ $count -lt $wait_timer ]; do
        count=$(( count + 1 ))
        sleep 60s
        if ( sudo -u rcb -- ssh ${SSH_OPTS} crowbar@${CROWBAR} "${cmd} proposal show ${PROPOSAL_NAME} | grep crowbar-status | grep success" ); then
            echo "${service} proposal sucessfully applied"
            break
        fi
        if [ $count == $wait_timer ]; then
            echo "${service} proposal not applied"
            cleanup
            exit 1
        fi
    done
}


SSH_OPTS="-n -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

# Prepare bastion interface/iptables
echo "Setting up iptables and system forwarding for eth0.."
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
sed -i 's/^#net.ipv4.ip_forward/net.ipv4.ip_forward/' /etc/sysctl.conf
sysctl -p /etc/sysctl.conf

# Check for/Generate ssh key
echo "Generating ssh keys.."
if [ ! -f /home/${SUDO_USER}/.ssh/id_rsa.pub ]; then
    ssh-keygen -t rsa -f /home/${SUDO_USER}/.ssh/id_rsa -N '' -q
fi

## Install ssh key
chown -R ${SUDO_USER}:${SUDO_USER} /home/${SUDO_USER}/.ssh
chmod -R go-rwx /home/${SUDO_USER}/.ssh

# Remove old deployrc
if [ -f .deployrc ]; then
    rm -rf .deployrc
fi

# Parse json file, extract bastion/pxeapp/infra/crowbar addresses, export to .deployrc
echo "Extracting json values to environment variables.."
for i in BASTION PXEAPP INFRA INFRA_MAC INFRA_DRAC CROWBAR NETMASK GATEWAY NAMESERVER CBFQDN NODECOUNT CONT_MAC; do  python -c "import json; import os; data = open('env.json');json_data = json.load(data); data.close(); print json_data['attributes']['network']['reserved']['$i'.lower()]" | echo "export $i=`awk '{print $0}'`" >> .deployrc; done

# Source .deployrc
source .deployrc

# Install necessary packages
echo "Installing Packages.."
apt-get install -y libvirt-bin ipmitool curl qemu-kvm

# Download pxeappliance image
echo "Downloading pxeappliance from cloud files.."
if [ ! -f /home/${SUDO_USER}/pxeappliance-dist.qcow2 ]; then
    curl -o /home/${SUDO_USER}/pxeappliance-dist.qcow2 ${PXE_IMAGE_URL}
fi
if [ ! -f /home/${SUDO_USER}/pxeappliance.xml ]; then
    curl -o /home/${SUDO_USER}/pxeappliance.xml ${PXE_XML_URL}
fi

echo "Moving pxeappliance into place."
mkdir -p /opt/rcb
mkdir -p /mnt/pxeapp
cp /home/${SUDO_USER}/pxeappliance-dist.qcow2 /opt/rcb/
cp /home/${SUDO_USER}/pxeappliance.xml /opt/rcb/

# Mount image
echo "Mounting pxeppliance qcow.."
modprobe nbd max_part=8
modprobe kvm
modprobe kvm_intel
qemu-nbd -c /dev/nbd0 /opt/rcb/pxeappliance-dist.qcow2

sleep 3

if [ ! -e /dev/nbd0p1 ]; then
    partprobe /dev/nbd0
fi

sleep 3

mount /dev/nbd0p1 /mnt/pxeapp

# Fixup preseed and pxelinux.cfg/defaul to match environment
echo "Modifying infra node preseed with environment values.."
sed -i "s/<nameserver>/${NAMESERVER}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
sed -i "s/<infra ip>/${INFRA}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
sed -i "s/<netmask>/${NETMASK}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
sed -i "s/<gateway>/${GATEWAY}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
PUBKEY=`cat /home/${SUDO_USER}/.ssh/id_rsa.pub`
sed -i "/^#d-i preseed\/late_command string/a d-i preseed\/late_command string wget http:\/\/${PXEAPP}\/post_install.sh -O \/target\/root\/post_install.sh; chmod a+x \/target\/root\/post_install.sh; chroot \/target \/root\/post_install.sh" /mnt/pxeapp/var/www/preseed.txt
sed -i "s/<pxeapp>/${PXEAPP}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default

# Set Rackspace DNS in pxeappliance
echo "Setting pxeappliance nameserver.."
echo "nameserver 64.39.2.170" >> /mnt/pxeapp/etc/resolv.conf

# Create post_install.sh and move to apache dir for later..
echo "Creating post_install script for admin/infra node.."
cat >post_install.sh << EOF
#!/bin/bash
mkdir /home/rcb/.ssh
chmod -R 700 /home/rcb/.ssh
echo '${PUBKEY}' >> /home/rcb/.ssh/authorized_keys
chmod -R 600 /home/rcb/.ssh/authorized_keys
chown -R rcb:rcb /home/rcb/.ssh/
wget -O /home/rcb/.deployrc http://${PXEAPP}/deployrc
wget -O /home/rcb/install-crowbar http://${PXEAPP}/install-crowbar
wget -O /home/rcb/network.json http://${PXEAPP}/network.json
wget -O /home/rcb/firstboot.sh http://${PXEAPP}/firstboot.sh
chown rcb:rcb /home/rcb/install-crowbar
chmod ug+x /home/rcb/install-crowbar
chmod -R a+r /etc/gemrc
sed -i '/^exit/i /bin/bash /home/rcb/install-crowbar >> /var/log/install-crowbar.log 2>&1' /etc/rc.local
cat > /etc/network/interfaces <<EOFNET
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet manual
    post-up ifconfig eth0 up

auto br0
iface br0 inet static
    address ${INFRA}
    netmask ${NETMASK}
    gateway ${GATEWAY}
    bridge_ports eth0
    bridge_stp off
    bridge_maxwait 0
    bridge_fd 0

EOFNET
echo 'nameserver ${NAMESERVER}' > /etc/resolv.conf
EOF

# Crowbar Appliance firstboot
echo "Generating firstboot script for crowbar head node.."
cat >firstboot.sh << EOF
#!/bin/bash
mkdir /home/crowbar/.ssh
echo '${PUBKEY}' >> /home/crowbar/.ssh/authorized_keys
chown -R 1000:1000 /home/crowbar/.ssh
chmod -R 0700 /home/crowbar/.ssh

# INSTALL CROWBAR
./tftpboot/ubuntu_dvd/extra/install ${CBFQDN}

# REMOVE FIRSTBOOT
chmod -R a+r /etc/gemrc
sed -i '\/bin\/bash \/home\/crowbar\/firstboot.sh/d' /etc/rc.local
EOF

echo "Putting scripts in place.."
cp post_install.sh /mnt/pxeapp/var/www/post_install.sh
cp network.json /mnt/pxeapp/var/www/network.json
cp firstboot.sh /mnt/pxeapp/var/www/firstboot.sh

# Copy crowbar install script to the apache dir for later..
cp .deployrc /mnt/pxeapp/var/www/deployrc
cp install-crowbar /mnt/pxeapp/var/www/install-crowbar

# Fix up the install-crowbar
sed -i "s/<nameserver>/${NAMESERVER}/" /mnt/pxeapp/var/www/install-crowbar
sed -i "s/<crowbar>/${CROWBAR}/" /mnt/pxeapp/var/www/install-crowbar
sed -i "s/<netmask>/${NETMASK}/" /mnt/pxeapp/var/www/install-crowbar
sed -i "s/<gateway>/${GATEWAY}/" /mnt/pxeapp/var/www/install-crowbar

# Insert eth0 configuration into /etc/network/interfaces
echo "Modifying pxeappliance network interfaces.."
cat /mnt/pxeapp/etc/network/interfaces > /tmp/interfaces
cat >/tmp/interfaces << EOF
auto eth0
iface eth0 inet static
    address ${PXEAPP}
    netmask ${NETMASK}
    gateway ${BASTION}
EOF
mv /tmp/interfaces /mnt/pxeapp/etc/network/interfaces
rm -rf /tmp/interfaces

# Create dnsmasq.conf
echo "Setting up pxeappliance dnsmasq.."
cat >/tmp/dnsmasq.conf << EOF
dhcp-ignore=tag:!known
dhcp-ignore-names

dhcp-range=${PXEAPP},static,24h,tag:known
dhcp-option=3,${GATEWAY}
dhcp-option-force=208,f1:00:74:7e
dhcp-option-force=210,/
dhcp-option-force=211,30
dhcp-boot=pxelinux.0

enable-tftp
tftp-root=/srv/tftproot

dhcp-host=${INFRA_MAC},infranode,${INFRA}
EOF
mv /tmp/dnsmasq.conf /mnt/pxeapp/etc/dnsmasq.conf
rm -rf /tmp/dnsmasq.conf

# Turn off ipv6 (seems to cause some dnsmasq problems)
cat > /mnt/pxeapp/etc/modprobe.d/00local.conf <<EOF
install ipv6 /bin/true
alias net-pf-10 off
alias ipv6 off
EOF

sed -i /mnt/pxeapp/boot/grub/grub.cfg -e 's#quiet$#quiet ipv6.disable=1#'

# Unmount modified image
echo "Unmounting pxeappliance image.."
umount /mnt/pxeapp
qemu-nbd -d /dev/nbd0

# Restart libvirt-bin to avoid hvm errors..
echo "Restart libvirt-bin.."
/etc/init.d/libvirt-bin restart

# Register domain / boot pxeapp
echo "Defining pxeappliance domain and starting appliance.."
# check if we have a lingering appliance and destroy
if  virsh dominfo pxeappliance >/dev/null 2>&1; then
    echo "pxeappliance already exists...destroying first"
    virsh destroy pxeappliance
    virsh undefine pxeappliance
fi

virsh define /opt/rcb/pxeappliance.xml
virsh start pxeappliance

echo "PXE boot admin/infra node.."
ipmi_pxeboot
sleep 10s

# WAIT FOR ADMIN NODE TO HAVE SSH
# This takes ~30 mins
echo "Waiting for admin node to be accessible.."
port_test "30" ${INFRA} "22"

# Destroy pxeappliance vm/domain
virsh destroy pxeappliance
virsh undefine pxeappliance

# WAIT FOR CROWBAR NODE TO BE RESPONSIVE
# Giving ~30 minutes
echo "Waiting for crowbar installation.."
port_test "30" ${CROWBAR} "3000"

# Since all nodes should be sitting in PXE we will wait a maximum of 30 minutes for all nodes to register
echo "Waiting for all crowbar managed nodes to register.."
count=1
while [ $count -lt 30 ]; do
    count=$((count +1))
    sleep 60s
    ELEMENTS=$(sudo -u ${SUDO_USER} -- ssh ${SSH_OPTS} ${CUSERNAME}@${CROWBAR} "/opt/dell/bin/crowbar_node_state -U ${CUSERNAME} -P ${CPASSWORD} status --no-ready | wc -l")
    if [ "$ELEMENTS" == "$NODECOUNT" ]; then
        break
    fi
    if [ $count -eq 30 ]; then
        echo "Some crowbar nodes did not come up."
        cleanup
        exit 1
    fi
done

##################################################
# BEGIN OPENSTACK PROPOSALS
##################################################

##################################################
# Push MYSQL Proposal
crowbar_proposal "mysql" "create"
crowbar_proposal_edit "mysql" "$CONT_MAC"
crowbar_proposal "mysql" "commit"
crowbar_proposal_status "mysql" 30
##################################################

##################################################
# Push the Keystone Proposal
crowbar_proposal "keystone" "create"
crowbar_proposal_edit "keystone" "$CONT_MAC"
crowbar_proposal "keystone" "commit"
crowbar_proposal_status "keystone"
##################################################

##################################################
# Push the Glance Proposal
crowbar_proposal "glance" "create"
crowbar_proposal_edit "glance" "$CONT_MAC"
crowbar_proposal "glance" "commit"
crowbar_proposal_status "glance"
##################################################

##################################################
# Push the Nova Proposal
crowbar_proposal "nova" "create"
crowbar_proposal "nova" "commit"
crowbar_proposal_status "nova" 30
##################################################

##################################################
# Push the Dash Proposal
crowbar_proposal "nova_dashboard" "create"
crowbar_proposal_edit "nova_dashboard" "$CONT_MAC"
crowbar_proposal "nova_dashboard" "commit"
crowbar_proposal_status "nova_dashboard"
##################################################

## Cleanup
cleanup
	{
	"id": "rcb deployment setup",
	"description": "Setup for deploy script",
	"attributes": {
	"network": {
	"reserved": {
	"bastion": "172.31.0.5",
	"pxeapp": "172.31.0.6",
	"infra": "172.31.0.9",
	"infra_mac": "60:eb:69:6e:f0:31",
	"infra_drac": "172.31.0.4",
	"cont_mac": "aa:bb:cc:dd:ee:ff",
	"crowbar": "172.31.0.10",
	"netmask": "255.255.255.0",
	"gateway": "172.31.0.5",
	"nameserver": "64.39.2.170",
	"cbfqdn": "lab.rcb.me",
	"nodecount": "14"
	}
	}
	}
	}
	#!/bin/bash

	## TURN ON FOR DEBUGGING
	# set -e
	# set -x

	## URL's for downloading the PXEAPPLIANCE
	PXE_IMAGE_URL=${PXE_IMAGE_URL:-http://c271871.r71.cf1.rackcdn.com/pxeappliance_gold.qcow2}
	PXE_XML_URL=${PXE_XML_URL:-http://c271871.r71.cf1.rackcdn.com/pxeappliance.xml}

	# FOR CROWBAR PROPOSALS
	PROPOSAL_NAME="openstack"

	## Cleanup function for all events
	function cleanup() {
	echo "Exiting..."
	virsh destroy pxeappliance
	virsh undefine pxeappliance
	qemu-nbd -d /dev/nbd0
	rm -rf /var/lock/shep_protection.lock
	rm -rf /opt/rcb/*
	exit $?
	}

	## SET INTERRUPT EXIT
	trap cleanup SIGINT

	## GRAB DRAC AND CROWBAR CREDENTIALS:
	## NOTE: You must create a .creds file with DRAC USER and PASSWORD
	## File should contain DUSERNAME, DPASSWORD, CUSERNAME, CPASSWORD
	DIR="$( cd "$( dirname "$0" )" && pwd )"
	if [ -f $DIR/.creds ]; then
	source .creds
	else
	echo "Missing creds file."
	cleanup
	exit $?
	fi

	## Ensure i'm not already running (Shep protection)
	if [ -f /var/lock/shep_protection.lock ]; then
	echo "Running already"
	exit 1
	else
	touch /var/lock/shep_protection.lock
	fi

	## Simple function for ipmi boot of infra node
	function ipmi_pxeboot() {
	POWERSTATE=`ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis status \| grep System \| awk '{print $4}'`
	if [ "$POWERSTATE" == 'on' ]; then
	for i in $(seq 1 5); do
	/usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis bootdev pxe
	/usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis power cycle
	done
	else
	for i in $(seq 1 5); do
	/usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis bootdev pxe
	/usr/bin/ipmitool -H ${INFRA_DRAC} -U $DUSERNAME -P $DPASSWORD chassis power on
	done
	fi
	}

	## Given x minutes, a node, and a port test response for x minutes. If the port doesn't become available in said time
	## exit and clean up.
	function port_test() {
	# $1 - Minutes to wait
	# $2 - NODE to test
	# $3 - Port to test (netcat)
	maxwait=$1
	node=$2
	port=$3
	count=1
	while [ $count -lt $maxwait ]; do
	count=$(( count + 1 ))
	sleep 60s
	if ( nc $node $port -w 1 -q 0 < /dev/null ); then
	break
	fi
	if [ $count -eq $maxwait ]; then
	echo "Admin/Infra node is not network accessible"
	cleanup
	exit 1
	fi
	done
	}

	## Given a service name and a action create/commit a crowbar proposal. This in effect installs service software on node
	## determined by crowbar discovery.
	function crowbar_proposal() {
	# $1 - Service Name
	# $2 - Action (create\|commit)
	service=$1
	action=$2
	cmd="/opt/dell/bin/crowbar_${service} -U ${CUSERNAME} -P ${CPASSWORD}"
	echo "Executing crowbar_proposal using:"
	echo " Service: ${service}"
	echo " Action: ${action}"

	if ! ( sudo -u rcb -- ssh ${SSH_OPTS} crowbar@${CROWBAR} "${cmd} proposal ${action} ${PROPOSAL_NAME}" ); then
	echo "Unable to ${action} the ${service} Proposal"
	cleanup
	exit 1
	fi
	}

	function crowbar_proposal_edit() {

	# $1 - Service name
	# $2 - second infra mac
	service=$1
	mac=$2
	cmd="/opt/dell/bin/crowbar_${service} -U ${CUSERNAME} -P ${CPASSWORD}"
	echo "Editing crowbar_proposal using:"
	echo " Service: ${service}"
	echo " Action: edit"

	# make the mac address into a crowbar friendly hostname
	target_host="$(echo d${mac}.${CBFQDN} \| sed -e s/\:/\-/)"

	# urgh need to fix so it dumps out on remote node rather than locally
	sudo -u rcb -- ssh ${SSH_OPTS} crowbar@${CROWBAR} "${cmd} proposal show ${PROPOSAL_NAME} > /tmp/$service.json; \
	current_host=$(grep $CBFQDN $service.json\|sed -e 's/^[ \t]*//'\|sed -e 's/\"//g' ) ; \
	sed -i -e s/$current_host/$target_host/ $service.json; \
	${cmd} proposal edit $service --file=/tmp/$service.json"

	}



	## Given service name and time to wat check the status of the above proposal until Active.
	function crowbar_proposal_status() {
	# $1 - Service Name
	# $2 - Wait Time
	service=$1
	wait_timer=${2:-15} # Default to 15 minutes if no wait_time provided
	cmd="/opt/dell/bin/crowbar_${service} -U ${CUSERNAME} -P ${CPASSWORD}"
	echo "Executing crowbar_proposal using:"
	echo " Service: ${service}"
	echo " Wait Time: ${wait_timer}"

	count=1
	while [ $count -lt $wait_timer ]; do
	count=$(( count + 1 ))
	sleep 60s
	if ( sudo -u rcb -- ssh ${SSH_OPTS} crowbar@${CROWBAR} "${cmd} proposal show ${PROPOSAL_NAME} \| grep crowbar-status \| grep success" ); then
	echo "${service} proposal sucessfully applied"
	break
	fi
	if [ $count == $wait_timer ]; then
	echo "${service} proposal not applied"
	cleanup
	exit 1
	fi
	done
	}


	SSH_OPTS="-n -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

	# Prepare bastion interface/iptables
	echo "Setting up iptables and system forwarding for eth0.."
	iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
	sed -i 's/^#net.ipv4.ip_forward/net.ipv4.ip_forward/' /etc/sysctl.conf
	sysctl -p /etc/sysctl.conf

	# Check for/Generate ssh key
	echo "Generating ssh keys.."
	if [ ! -f /home/${SUDO_USER}/.ssh/id_rsa.pub ]; then
	ssh-keygen -t rsa -f /home/${SUDO_USER}/.ssh/id_rsa -N '' -q
	fi

	## Install ssh key
	chown -R ${SUDO_USER}:${SUDO_USER} /home/${SUDO_USER}/.ssh
	chmod -R go-rwx /home/${SUDO_USER}/.ssh

	# Remove old deployrc
	if [ -f .deployrc ]; then
	rm -rf .deployrc
	fi

	# Parse json file, extract bastion/pxeapp/infra/crowbar addresses, export to .deployrc
	echo "Extracting json values to environment variables.."
	for i in BASTION PXEAPP INFRA INFRA_MAC INFRA_DRAC CROWBAR NETMASK GATEWAY NAMESERVER CBFQDN NODECOUNT CONT_MAC; do python -c "import json; import os; data = open('env.json');json_data = json.load(data); data.close(); print json_data['attributes']['network']['reserved']['$i'.lower()]" \| echo "export $i=`awk '{print $0}'`" >> .deployrc; done

	# Source .deployrc
	source .deployrc

	# Install necessary packages
	echo "Installing Packages.."
	apt-get install -y libvirt-bin ipmitool curl qemu-kvm

	# Download pxeappliance image
	echo "Downloading pxeappliance from cloud files.."
	if [ ! -f /home/${SUDO_USER}/pxeappliance-dist.qcow2 ]; then
	curl -o /home/${SUDO_USER}/pxeappliance-dist.qcow2 ${PXE_IMAGE_URL}
	fi
	if [ ! -f /home/${SUDO_USER}/pxeappliance.xml ]; then
	curl -o /home/${SUDO_USER}/pxeappliance.xml ${PXE_XML_URL}
	fi

	echo "Moving pxeappliance into place."
	mkdir -p /opt/rcb
	mkdir -p /mnt/pxeapp
	cp /home/${SUDO_USER}/pxeappliance-dist.qcow2 /opt/rcb/
	cp /home/${SUDO_USER}/pxeappliance.xml /opt/rcb/

	# Mount image
	echo "Mounting pxeppliance qcow.."
	modprobe nbd max_part=8
	modprobe kvm
	modprobe kvm_intel
	qemu-nbd -c /dev/nbd0 /opt/rcb/pxeappliance-dist.qcow2

	sleep 3

	if [ ! -e /dev/nbd0p1 ]; then
	partprobe /dev/nbd0
	fi

	sleep 3

	mount /dev/nbd0p1 /mnt/pxeapp

	# Fixup preseed and pxelinux.cfg/defaul to match environment
	echo "Modifying infra node preseed with environment values.."
	sed -i "s/<nameserver>/${NAMESERVER}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
	sed -i "s/<infra ip>/${INFRA}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
	sed -i "s/<netmask>/${NETMASK}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
	sed -i "s/<gateway>/${GATEWAY}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default
	PUBKEY=`cat /home/${SUDO_USER}/.ssh/id_rsa.pub`
	sed -i "/^#d-i preseed\/late_command string/a d-i preseed\/late_command string wget http:\/\/${PXEAPP}\/post_install.sh -O \/target\/root\/post_install.sh; chmod a+x \/target\/root\/post_install.sh; chroot \/target \/root\/post_install.sh" /mnt/pxeapp/var/www/preseed.txt
	sed -i "s/<pxeapp>/${PXEAPP}/" /mnt/pxeapp/srv/tftproot/pxelinux.cfg/default

	# Set Rackspace DNS in pxeappliance
	echo "Setting pxeappliance nameserver.."
	echo "nameserver 64.39.2.170" >> /mnt/pxeapp/etc/resolv.conf

	# Create post_install.sh and move to apache dir for later..
	echo "Creating post_install script for admin/infra node.."
	cat >post_install.sh << EOF
	#!/bin/bash
	mkdir /home/rcb/.ssh
	chmod -R 700 /home/rcb/.ssh
	echo '${PUBKEY}' >> /home/rcb/.ssh/authorized_keys
	chmod -R 600 /home/rcb/.ssh/authorized_keys
	chown -R rcb:rcb /home/rcb/.ssh/
	wget -O /home/rcb/.deployrc http://${PXEAPP}/deployrc
	wget -O /home/rcb/install-crowbar http://${PXEAPP}/install-crowbar
	wget -O /home/rcb/network.json http://${PXEAPP}/network.json
	wget -O /home/rcb/firstboot.sh http://${PXEAPP}/firstboot.sh
	chown rcb:rcb /home/rcb/install-crowbar
	chmod ug+x /home/rcb/install-crowbar
	chmod -R a+r /etc/gemrc
	sed -i '/^exit/i /bin/bash /home/rcb/install-crowbar >> /var/log/install-crowbar.log 2>&1' /etc/rc.local
	cat > /etc/network/interfaces <<EOFNET
	auto lo
	iface lo inet loopback

	auto eth0
	iface eth0 inet manual
	post-up ifconfig eth0 up

	auto br0
	iface br0 inet static
	address ${INFRA}
	netmask ${NETMASK}
	gateway ${GATEWAY}
	bridge_ports eth0
	bridge_stp off
	bridge_maxwait 0
	bridge_fd 0

	EOFNET
	echo 'nameserver ${NAMESERVER}' > /etc/resolv.conf
	EOF

	# Crowbar Appliance firstboot
	echo "Generating firstboot script for crowbar head node.."
	cat >firstboot.sh << EOF
	#!/bin/bash
	mkdir /home/crowbar/.ssh
	echo '${PUBKEY}' >> /home/crowbar/.ssh/authorized_keys
	chown -R 1000:1000 /home/crowbar/.ssh
	chmod -R 0700 /home/crowbar/.ssh

	# INSTALL CROWBAR
	./tftpboot/ubuntu_dvd/extra/install ${CBFQDN}

	# REMOVE FIRSTBOOT
	chmod -R a+r /etc/gemrc
	sed -i '\/bin\/bash \/home\/crowbar\/firstboot.sh/d' /etc/rc.local
	EOF

	echo "Putting scripts in place.."
	cp post_install.sh /mnt/pxeapp/var/www/post_install.sh
	cp network.json /mnt/pxeapp/var/www/network.json
	cp firstboot.sh /mnt/pxeapp/var/www/firstboot.sh

	# Copy crowbar install script to the apache dir for later..
	cp .deployrc /mnt/pxeapp/var/www/deployrc
	cp install-crowbar /mnt/pxeapp/var/www/install-crowbar

	# Fix up the install-crowbar
	sed -i "s/<nameserver>/${NAMESERVER}/" /mnt/pxeapp/var/www/install-crowbar
	sed -i "s/<crowbar>/${CROWBAR}/" /mnt/pxeapp/var/www/install-crowbar
	sed -i "s/<netmask>/${NETMASK}/" /mnt/pxeapp/var/www/install-crowbar
	sed -i "s/<gateway>/${GATEWAY}/" /mnt/pxeapp/var/www/install-crowbar

	# Insert eth0 configuration into /etc/network/interfaces
	echo "Modifying pxeappliance network interfaces.."
	cat /mnt/pxeapp/etc/network/interfaces > /tmp/interfaces
	cat >/tmp/interfaces << EOF
	auto eth0
	iface eth0 inet static
	address ${PXEAPP}
	netmask ${NETMASK}
	gateway ${BASTION}
	EOF
	mv /tmp/interfaces /mnt/pxeapp/etc/network/interfaces
	rm -rf /tmp/interfaces

	# Create dnsmasq.conf
	echo "Setting up pxeappliance dnsmasq.."
	cat >/tmp/dnsmasq.conf << EOF
	dhcp-ignore=tag:!known
	dhcp-ignore-names

	dhcp-range=${PXEAPP},static,24h,tag:known
	dhcp-option=3,${GATEWAY}
	dhcp-option-force=208,f1:00:74:7e
	dhcp-option-force=210,/
	dhcp-option-force=211,30
	dhcp-boot=pxelinux.0

	enable-tftp
	tftp-root=/srv/tftproot

	dhcp-host=${INFRA_MAC},infranode,${INFRA}
	EOF
	mv /tmp/dnsmasq.conf /mnt/pxeapp/etc/dnsmasq.conf
	rm -rf /tmp/dnsmasq.conf

	# Turn off ipv6 (seems to cause some dnsmasq problems)
	cat > /mnt/pxeapp/etc/modprobe.d/00local.conf <<EOF
	install ipv6 /bin/true
	alias net-pf-10 off
	alias ipv6 off
	EOF

	sed -i /mnt/pxeapp/boot/grub/grub.cfg -e 's#quiet$#quiet ipv6.disable=1#'

	# Unmount modified image
	echo "Unmounting pxeappliance image.."
	umount /mnt/pxeapp
	qemu-nbd -d /dev/nbd0

	# Restart libvirt-bin to avoid hvm errors..
	echo "Restart libvirt-bin.."
	/etc/init.d/libvirt-bin restart

	# Register domain / boot pxeapp
	echo "Defining pxeappliance domain and starting appliance.."
	# check if we have a lingering appliance and destroy
	if virsh dominfo pxeappliance >/dev/null 2>&1; then
	echo "pxeappliance already exists...destroying first"
	virsh destroy pxeappliance
	virsh undefine pxeappliance
	fi

	virsh define /opt/rcb/pxeappliance.xml
	virsh start pxeappliance

	echo "PXE boot admin/infra node.."
	ipmi_pxeboot
	sleep 10s

	# WAIT FOR ADMIN NODE TO HAVE SSH
	# This takes ~30 mins
	echo "Waiting for admin node to be accessible.."
	port_test "30" ${INFRA} "22"

	# Destroy pxeappliance vm/domain
	virsh destroy pxeappliance
	virsh undefine pxeappliance

	# WAIT FOR CROWBAR NODE TO BE RESPONSIVE
	# Giving ~30 minutes
	echo "Waiting for crowbar installation.."
	port_test "30" ${CROWBAR} "3000"

	# Since all nodes should be sitting in PXE we will wait a maximum of 30 minutes for all nodes to register
	echo "Waiting for all crowbar managed nodes to register.."
	count=1
	while [ $count -lt 30 ]; do
	count=$((count +1))
	sleep 60s
	ELEMENTS=$(sudo -u ${SUDO_USER} -- ssh ${SSH_OPTS} ${CUSERNAME}@${CROWBAR} "/opt/dell/bin/crowbar_node_state -U ${CUSERNAME} -P ${CPASSWORD} status --no-ready \| wc -l")
	if [ "$ELEMENTS" == "$NODECOUNT" ]; then
	break
	fi
	if [ $count -eq 30 ]; then
	echo "Some crowbar nodes did not come up."
	cleanup
	exit 1
	fi
	done

	##################################################
	# BEGIN OPENSTACK PROPOSALS
	##################################################

	##################################################
	# Push MYSQL Proposal
	crowbar_proposal "mysql" "create"
	crowbar_proposal_edit "mysql" "$CONT_MAC"
	crowbar_proposal "mysql" "commit"
	crowbar_proposal_status "mysql" 30
	##################################################

	##################################################
	# Push the Keystone Proposal
	crowbar_proposal "keystone" "create"
	crowbar_proposal_edit "keystone" "$CONT_MAC"
	crowbar_proposal "keystone" "commit"
	crowbar_proposal_status "keystone"
	##################################################

	##################################################
	# Push the Glance Proposal
	crowbar_proposal "glance" "create"
	crowbar_proposal_edit "glance" "$CONT_MAC"
	crowbar_proposal "glance" "commit"
	crowbar_proposal_status "glance"
	##################################################

	##################################################
	# Push the Nova Proposal
	crowbar_proposal "nova" "create"
	crowbar_proposal "nova" "commit"
	crowbar_proposal_status "nova" 30
	##################################################

	##################################################
	# Push the Dash Proposal
	crowbar_proposal "nova_dashboard" "create"
	crowbar_proposal_edit "nova_dashboard" "$CONT_MAC"
	crowbar_proposal "nova_dashboard" "commit"
	crowbar_proposal_status "nova_dashboard"
	##################################################

	## Cleanup
	cleanup