Creating a highly-available Chef cluster with 3 backends and 2 frontends in an AWS VPC

create_chef_cluster.sh

#!/bin/bash -l
set -e -x -u
# Below is a script to create and configure the chef cluster, provided as a reference and documentation for the steps.
# The steps are documented in the chef cluster docs: https://docs.chef.io/install_server_ha.html
# Created by Innovid.com

region=$1
if [ -z "${region}" ]; then
        echo "Please specify a region"
        exit 1
fi

###############################################
#           Define Your Variables             #
###############################################

backend_a_name="chefbackend-1.example.com"
backend_b_name="chefbackend-2.example.com"
backend_c_name="chefbackend-3.example.com"
frontend_a_name="cheffrontend-1.example.com"
frontend_b_name="cheffrontend-2.example.com"


backend_package="https://packages.chef.io/files/stable/chef-backend/1.4.6/ubuntu/16.04/chef-backend_1.4.6-1_amd64.deb"
frontend_package="https://packages.chef.io/files/stable/chef-server/12.15.8/ubuntu/16.04/chef-server-core_12.15.8-1_amd64.deb"
ssh_key_path="${HOME}/.ssh/id_rsa"
user="${USER}"

##########################################
#       Get Nodes IPs from AWS           #
##########################################

echo "Getting public DNSs"
frontend_a_pub=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${frontend_a_name}" --query 'Reservations[*].Instances[*].PublicDnsName' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-frontend-1: "${frontend_a_pub}""
frontend_b_pub=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${frontend_b_name}" --query 'Reservations[*].Instances[*].PublicDnsName' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-frontend-2: "${frontend_b_pub}""
backend_a_pub=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${backend_a_name}" --query 'Reservations[*].Instances[*].PublicDnsName' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-backend-1: "${backend_a_pub}""
backend_b_pub=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${backend_b_name}" --query 'Reservations[*].Instances[*].PublicDnsName' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-backend-2: "${backend_b_pub}""
backend_c_pub=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${backend_c_name}" --query 'Reservations[*].Instances[*].PublicDnsName' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-backend-3: "${backend_c_pub}""

echo "Getting private IPs"
frontend_a_private=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${frontend_a_name}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-frontend-1: "${frontend_a_private}""
frontend_b_private=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${frontend_b_name}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-frontend-2: "${frontend_b_private}""
backend_a_private=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${backend_a_name}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-backend-1: "${backend_a_private}""
backend_b_private=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${backend_b_name}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-backend-2: "${backend_b_private}""
backend_c_private=$(aws ec2 describe-instances --region ${region} --filters "Name=tag-value,Values=${backend_c_name}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text | awk '{gsub(/ */,"",$1); print $1}')
echo "chef-backend-3: "${backend_c_private}""

private_cidrs="\"${backend_a_private}/32\", \"${backend_b_private}/32\", \"${backend_c_private}/32\", \"${frontend_a_private}/32\",\"${frontend_b_private}/32\""
##########################################
#         Create leader: Backend-1       #
##########################################
echo "Connecting to chef-backend-1 to configure and generate server secrets"
ssh -To "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_a_pub} << EOF

wget ${backend_package} -qP /tmp/
yes yes | sudo dpkg -i /tmp/*.deb
sudo mkdir -p /etc/chef-backend/
echo "publish_address '${backend_a_private}' # ip address of this backend box" | sudo tee -a /etc/chef-backend/chef-backend.rb
echo "postgresql.md5_auth_cidr_addresses = [\"samehost\", \"samenet\", ${private_cidrs}]" | sudo tee -a /etc/chef-backend/chef-backend.rb
sudo chef-backend-ctl create-cluster --yes --accept-license
sudo cp /etc/chef-backend/chef-backend-secrets.json /tmp/
sudo chmod 777 /tmp/chef-backend-secrets.json
EOF

##########################################
#         Copy secrets to backends       #
##########################################
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_a_pub}:/tmp/chef-backend-secrets.json /tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} /tmp/chef-backend-secrets.json ${user}@${backend_b_pub}:/tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} /tmp/chef-backend-secrets.json ${user}@${backend_c_pub}:/tmp/

##########################################
#         Create follower: Backend-2     #
##########################################
echo "Connecting to chef-backend-2 to configure and join cluster"
ssh -To "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_b_pub} << EOF

wget ${backend_package} -qP /tmp/
yes yes | sudo dpkg -i /tmp/*.deb
sudo mkdir -p /etc/chef-backend/
echo "publish_address '${backend_b_private}' # ip address of this backend box" | sudo tee -a /etc/chef-backend/chef-backend.rb
echo "postgresql.md5_auth_cidr_addresses = [\"samehost\", \"samenet\", ${private_cidrs}]" | sudo tee -a /etc/chef-backend/chef-backend.rb
sudo chef-backend-ctl join-cluster ${backend_a_private} -s /tmp/chef-backend-secrets.json --yes --accept-license
sudo rm -f /tmp/chef-*
EOF

##########################################
#         Create follower: Backend-3     #
##########################################
echo "Connecting to chef-backend-3 to configure and join cluster"
ssh -To "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_c_pub} << EOF

wget ${backend_package} -qP /tmp/
yes yes |sudo dpkg -i /tmp/*.deb
sudo mkdir -p /etc/chef-backend/
echo "publish_address '${backend_c_private}' # ip address of this backend box" | sudo tee -a /etc/chef-backend/chef-backend.rb
echo "postgresql.md5_auth_cidr_addresses = [\"samehost\", \"samenet\", ${private_cidrs}]" | sudo tee -a /etc/chef-backend/chef-backend.rb
sudo chef-backend-ctl join-cluster ${backend_a_private} -s /tmp/chef-backend-secrets.json --yes --accept-license
sudo rm -f /tmp/chef-*
EOF

##########################################
# Generating server config for frontends #
##########################################
echo "Connecting to chef-backend-1 to generate server configuration for frontend"
ssh -To "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_a_pub} << EOF
sudo chef-backend-ctl gen-server-config ${frontend_a_private} -f /tmp/chef-server.rb.FE1
sudo chmod 777 /tmp/chef-server.rb.FE1
EOF

##########################################
#   Copying server config to frontends   #
##########################################
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_a_pub}:/tmp/chef-server.rb.FE1 /tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} /tmp/chef-server.rb.FE1 ${user}@${frontend_a_pub}:/tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} /tmp/chef-server.rb.FE1 ${user}@${frontend_b_pub}:/tmp/

##########################################
#         Create: Frontend-1             #
##########################################
echo "Connecting to chef-frontend-1 to configure chef"
ssh -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${frontend_a_pub} bash << EOF

wget ${frontend_package} -qP /tmp
yes yes | sudo dpkg -i /tmp/*.deb
sudo mkdir -p /etc/opscode/

sudo mv /tmp/chef-server.rb.FE1 /etc/opscode/chef-server.rb
sudo chef-server-ctl reconfigure

sudo cp /etc/opscode/private-chef-secrets.json /tmp
sudo cp /var/opt/opscode/upgrades/migration-level /tmp
sudo chmod 777 /tmp/private-chef-secrets.json /tmp/migration-level
EOF

##########################################
#         Copy config to frontend-b      #
##########################################
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${frontend_a_pub}:/tmp/private-chef-secrets.json /tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${frontend_a_pub}:/tmp/migration-level /tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} /tmp/private-chef-secrets.json ${user}@${frontend_b_pub}:/tmp/
scp -o "StrictHostKeyChecking no" -i ${ssh_key_path} /tmp/migration-level ${user}@${frontend_b_pub}:/tmp/

##########################################
#         Create: Frontend-2             #
##########################################
echo "Connecting to chef-frontend-2 to configure chef"
ssh -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${frontend_b_pub} bash << EOF

wget ${frontend_package} -qP /tmp
yes yes | sudo dpkg -i /tmp/*.deb
sudo mkdir -p /etc/opscode/
sudo mkdir -p /var/opt/opscode/upgrades/

sudo mv /tmp/private-chef-secrets.json /etc/opscode/
sudo mv /tmp/migration-level /var/opt/opscode/upgrades/
sudo chown root:root /etc/opscode/private-chef-secrets.json /var/opt/opscode/upgrades/migration-level
sudo touch /var/opt/opscode/bootstrapped
sudo chef-server-ctl reconfigure

sudo rm -f /tmp/chef-*
EOF

##########################################
#                Cleanup                 #
##########################################
# backend a:
echo "Connecting to chef-backend-1 to clean up"
ssh -To "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${backend_a_pub} << EOF
sudo rm -f /tmp/chef-*
EOF

# frontend a:
echo "Connecting to chef-frontend-1 to clean up"
ssh -o "StrictHostKeyChecking no" -i ${ssh_key_path} ${user}@${frontend_a_pub} bash << EOF
sudo rm -f /tmp/chef-*
EOF