AWS User script to bootstrap any instance

bootstrap_instance.sh

#!/bin/bash -x

# User script to bootstrap any instance.
# This will set up, array, and format every instance storage
# drive present in the instance (if any)
#
# The disk format/layout is read from an EC2 tag "DiskLayout"
# (defined in the cloudformation templates) and it defaults to
# "raid0". Possible values are:
# * raidN: grabs all available instance storage drives, creates a
#   single software raid device with RAID level N, mounts it under
#   /mnt and moves /usr, /var, /home to it (the EBS root volumes are
#   usually tiny)
# * jbod: grabs each available instance storage drives and mounts
#   it under separate subdirectories under /mnt. Currently unused
#   but it might come in handy

# Upgrade packages
DEBIAN_FRONTEND=noninteractive apt-get -y update;
DEBIAN_FRONTEND=noninteractive apt-get -y -f upgrade;

# Force date adjustment and restart cron
service ntp stop
ntpdate -s 0.ubuntu.pool.ntp.org
service ntp start
service cron restart

# will need the raid tools and the aws tools
DEBIAN_FRONTEND=noninteractive apt-get install -y mdadm awscli;

# We'll ask the metadata api for the block device mapping
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
function curly() {
  local uri="$1";
  curl -s "http://169.254.169.254/latest/meta-data/${uri}" && echo;
}

function get_tag() {
  local tag="$1";
  local default="${2:-}"

  # Get this instance's information from metadata api
  local instance_id="$( curly instance-id )";
  local az="$( curly placement/availability-zone )";
  # cut the last char from the AZ and you get the Region
  local region="${az%?}";

  # thanks to the magic of amazon tags
  # http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-tags.html
  local value=`aws --region=${region} ec2 describe-tags --output text --filters "Name=resource-id,Values=${instance_id}" | grep ${tag} | awk '{ print $NF }'`;
  if [ -z "$value" ]; then
    echo $default;
  else
    echo $value;
  fi;
}

function move_dirs() {
  # Since EC2 instance-store backed instances have only 10GB root
  # partitions, we want to mount important root directories to
  # a different (larger) mount point
  # See:
  # http://www.turnkeylinux.org/docs/using-instance-storage
  local target="$1";
  for dir in var usr home; do
    mv /${dir} ${target}/;
    mkdir -p /${dir};
    echo -e "${target}/${dir}\t/${dir}\tbind\tbind\t0 0" >> /etc/fstab
    mount /${dir};
  done;
}

# Set the hostname. These instructions are copied/adjusted from fabfile_puppet
hostname="$( get_tag Name localhost )";
if [ "$hostname" == "localhost" ]; then
  domain="localdomain";
else
  domain="example.com";
fi;
fqdn="${hostname}.${domain}";

hostname $hostname;
echo "$hostname" > /etc/hostname;

echo "# This file is automatically generated by our bootstrap script.
127.0.0.1 localhost.localdomain localhost
127.0.0.1 ${fqdn} ${hostname}

# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts" > /etc/hosts;

# These instances come with an entry in fstab for xvdb
# we want to remove this line so we can add the correct entries
# and unmount it for later manipulation
sed -i '/^\/dev\/xvdb/d' /etc/fstab;
umount /dev/xvdb;

disk_layout="$( get_tag DiskLayout raid0 )";

# our storage arrays
eph_drives=();
trim_drives=();

# Get all block devices as per Amazon's API and stick them in the array
for drive in $( curly block-device-mapping/ | grep ^ephemeral | paste -s -d' ' ); do
  # Get the actual device name translated to linux lingo i.e. sdb -> xvdb
  dev="$( curly block-device-mapping/${drive} | sed -e 's/s/xv/' )";
  eph_drives[${#eph_drives[*]}]="/dev/${dev}";

  # Does this device support TRIM? By rule amazon doesn't mix TRIM/non-TRIM
  # drives, so if you have at least one of them they are all TRIM
  # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes
  if lsblk -n -l -o "NAME,DISC-MAX" | grep ^${dev} | grep -v '0B'; then
    trim_drives[${#trim_drives[*]}]="/dev/${dev}";
  fi;
done;

# The fstab options change if there are TRIM capable drives
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ssd-instance-store.html#InstanceStoreTrimSupport
if [ -z "${trim_drives[*]}" ]; then
  fstab_opt="defaults,nobootwait";
else
  fstab_opt="defaults,discard";
fi;

# building a software RAID array given a tag (or default)
if [[ ! -z "${eph_drives[*]}" && "$disk_layout" == "raid"* ]]; then
  level="$( echo $disk_layout | sed -e 's/raid//' )";
  if [ ${#eph_drives[*]} -gt 1 ]; then
    device="/dev/md/eph:0";
    # create a software raid device and configure so it rebuilds properly every boot
    # this will force the array even if the drives haven't been partitioned or have
    # an existing file system
    mdadm --create $device --run --force --level=${level} --raid-devices=${#eph_drives[*]} ${eph_drives[*]};
    mdadm --detail --scan --verbose | sed -e "s/metadata=.*\ //" > /etc/mdadm/mdadm.conf;
  else
    # single drive
    device="${eph_drives[0]}";
  fi;

  # EXT4 all the things
  mkfs.ext4 -E nodiscard -F $device;
  # mount it properly
  mkdir -p /mnt;
  echo -e "${device}\t/mnt\text4\t${fstab_opt}\t0 0" >> /etc/fstab
  mount /mnt;
  move_dirs /mnt;
fi;

# JBOD-style configuration
if [[ ! -z "${eph_drives[*]}" && "$disk_layout" == "jbod" ]]; then
  first=true;
  for device in ${eph_drives[*]}; do
    name="$( basename $device )";
    mkfs.ext4 -E nodiscard -F $device;
    mkdir -p /mnt/${name};
    echo -e "${device}\t/mnt/${name}\text4\t${fstab_opt}\t0 0" >> /etc/fstab
    mount /mnt/${name};
    # move our stuff to the first device
    if $first; then
      move_dirs /mnt/${name};
      first=false;
    fi;
  done;
fi;

# Setup the puppet agent
puppetmaster="$( get_tag PuppetMaster puppet )"
DEBIAN_FRONTEND=noninteractive apt-get install -y puppet;
echo "# This file is automatically generated by our bootstrap script.
[main]
  logdir=/var/log/puppet
  vardir=/var/lib/puppet
  ssldir=/var/lib/puppet/ssl
  rundir=/var/run/puppet
  factpath=\$vardir/lib/facter
  templatedir=\$confdir/templates
  parser = future

[agent]
  server=${puppetmaster}
  report = true
  classfile = \$vardir/classes.txt
  localconfig = \$vardir/localconfig
  graph = true
" > /etc/puppet/puppet.conf;
puppet agent --enable;
puppet agent --no-daemonize --onetime --verbose --no-splay --parser=future --logdest="/var/log/puppet/firstrun.log";
service puppet restart;