Add ephemera storage to Amazon ECS-optimized AMIs (v1)

amazon-linux-1-user-data.txt

Content-Type: multipart/mixed; boundary="==BOUNDARY=="
MIME-Version: 1.0

--==BOUNDARY==
MIME-Version: 1.0
Content-Type: text/cloud-config
# this script ensures that empheral storage doesn't get mounted by cloud-init
mounts:
 - [ ephemeral0 ]


--==BOUNDARY==
MIME-Version: 1.0
Content-Type: text/cloud-boothook; charset="us-ascii"

#!/bin/bash +x
# an early running script to setup scripts that get triggered after the ECS init service is started
#
# Write the bootstrap script to /opt/devgrok and run it - so output definitely logged
mkdir -p /opt/devgrok
cat > /opt/devgrok/bootstrap-devgrok-init.sh <<- 'EOF'
#!/bin/bash -x
mkdir -p /opt/devgrok/ecs-init-scripts/
yum install -y aws-cli
aws s3 sync s3://devgrok-blog-files/amazon-ecs/ecs-init-scripts/ /opt/devgrok/ecs-init-scripts/
chmod +x /opt/devgrok/ecs-init-scripts/*.sh

# setup aws cloudwatch
/opt/devgrok/ecs-init-scripts/install-cw-agent.sh
[[ -e "/etc/init/devgrok-start-cloudwatch.conf" ]] || \
  cp /opt/devgrok/ecs-init-scripts/upstart-start-cloudwatch.conf /etc/init/devgrok-start-cloudwatch.conf

# setup storage
/opt/devgrok/ecs-init-scripts/docker-add-ephemeral.sh

initctl reload-configuration
EOF

chmod +x /opt/devgrok/bootstrap-devgrok-init.sh
/opt/devgrok/bootstrap-devgrok-init.sh 2>&1 | tee -a /var/log/bootstrap-devgrok-init.log

# this sets the maximum docker container size to 100GB
cloud-init-per once docker_options echo 'OPTIONS="${OPTIONS} --storage-opt dm.basesize=100G"' >> /etc/sysconfig/docker

--==BOUNDARY==--

amazon-linux-2-user-data.txt

Content-Type: multipart/mixed; boundary="==BOUNDARY=="
MIME-Version: 1.0

--==BOUNDARY==
MIME-Version: 1.0
Content-Type: text/cloud-config
# this script ensures that empheral storage doesn't get mounted by cloud-init
mounts:
 - [ ephemeral0 ]


--==BOUNDARY==
MIME-Version: 1.0
Content-Type: text/cloud-boothook; charset="us-ascii"

#!/bin/bash +x
# an early running script to setup scripts that get triggered after the ECS init service is started
#
# Write the bootstrap script to /opt/devgrok and run it - so output definitely logged
mkdir -p /opt/devgrok
cat > /opt/devgrok/bootstrap-devgrok-init.sh <<- 'EOF'
#!/bin/bash -x
mkdir -p /opt/devgrok/ecs-init-scripts/
yum install -y aws-cli
aws s3 sync s3://devgrok-blog-files/amazon-ecs/ecs-init-scripts/ /opt/devgrok/ecs-init-scripts/
chmod +x /opt/devgrok/ecs-init-scripts/*.sh

# setup aws cloudwatch
/opt/devgrok/ecs-init-scripts/install-cw-agent.sh
[[ -e "/etc/systemd/system/bootstrap-awslogs.service" ]] || \
  cp /opt/devgrok/ecs-init-scripts/systemd-bootstrap-awslogs.service /etc/systemd/system/bootstrap-awslogs.service

# setup storage
/opt/devgrok/ecs-init-scripts/docker-amzn-2-add-ephemeral.sh

EOF

chmod +x /opt/devgrok/bootstrap-devgrok-init.sh
/opt/devgrok/bootstrap-devgrok-init.sh 2>&1 | tee -a /var/log/bootstrap-devgrok-init.log


--==BOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/sh
# this runs later in the boot process
systemctl daemon-reload
systemctl enable bootstrap-awslogs.service
# the no-block is important as the ecs-init service may be dependant on the cloud-init boot process finishing first
systemctl start bootstrap-awslogs.service --no-block

--==BOUNDARY==--

docker-add-ephemeral.sh

#!/bin/bash -exu

# find ephemeral devices using the metadata service
# note: this will block loading until the metadata service is ready
find_ephemeral_devices() {
    echo "Querying metadata instance store for ephemeral volumes" >&2
    local DEVICE_NOT_FOUND=0
    for d in $(curl -s "http://169.254.169.254/latest/meta-data/block-device-mapping/" | grep ephemeral); do
       NAME=$(curl -s "http://169.254.169.254/latest/meta-data/block-device-mapping/$d")
       DEVICE_NAME="/dev/$NAME"
       echo "Detected ephemeral device $d corresponding to $DEVICE_NAME" >&2
       # if block device (-b)
       if [[ -b "$DEVICE_NAME" ]]; then
          echo "$DEVICE_NAME"
          DEVICE_NOT_FOUND=0
       else
          echo "Couldn't find device $DEVICE_NAME" >&2
          DEVICE_NOT_FOUND=1
       fi
    done
    return ${DEVICE_NOT_FOUND}
}

# on nvme instances the device names don't match what AWS reports, so just filter by model name
find_nvme_ephemeral() {
  lsblk -o KNAME,MODEL | grep "Amazon EC2 NVMe Instance Storage" | awk '{ print "/dev/"$1 }'
}

METADATA_DEVICES=$(find_ephemeral_devices)
NVME_DEVICES=$(find_nvme_ephemeral)

DEVS=/dev/xvdcz
[[ ! -z "$NVME_DEVICES" ]] && DEVS="$NVME_DEVICES $DEVS"
[[ ! -z "$METADATA_DEVICES" ]] && DEVS="$METADATA_DEVICES $DEVS"

echo "Updating docker storage devices to '$DEVS'"
sed -i -e "s;DEVS=.*;DEVS=\"${DEVS}\";g"  /etc/sysconfig/docker-storage-setup

docker-amzn-2-add-ephemeral.sh

#!/bin/bash
#--
# Copyright 2014-2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#++

#  Purpose:  This script sets up the storage for container runtimes.
#  Author:   Andy Grimm <agrimm@redhat.com>
#
#
# Trimmed down version to setup storage based of ext4+overlay2 on the current
# AMi provided by AWS (Amazon ECS-optimized Amazon Linux 2 AMI).
# This is instead of LVM/thin-pool+devicemapper configured in previous AMI
#    (Amazon ECS-optimized Amazon Linux AMI)
# This is a heavily cut=down version of:
# https://github.com/projectatomic/container-storage-setup/blob/master/container-storage-setup.sh
#
#  Author:   Chris Watts <watts.chris@gmail.com>
set -ex

# Partition type related
_MAX_MBR_SIZE_BYTES="2199023255040"

_TEMPDIR=$(mktemp --tmpdir -d)

# Keeps track of resolved device paths
_DEVS_RESOLVED=""

# Keeps track of if we created a volume group or not.
_VG_CREATED=

VG=docker
LV_NAME=docker-volumes
LV_MOUNT_PATH=/var/lib/docker/volumes
WIPE_SIGNATURES=true
CREATE_VG=true

# find devices which aren't the root partition, doesn't use metadata service to find ephemeral to speed up start-up time
function find_non_root_devices() {
  local root_part root_dev other_devices
  shopt -s nullglob
  root_part=$(lsblk -Pp | grep 'MOUNTPOINT="/"' |  sed -nr 's/NAME="([^"]+)".*/\1/p')
  root_dev=$(lsblk -pno pkname ${root_part})
  if [[ -z "$root_dev" ]]; then
    Fatal "Couldn't find root device, bailing"
  fi

  # filter out devices which aren't disks or are the root volume
  other_devices=$(lsblk -Pp | grep 'TYPE="disk"' | grep -v 'NAME="'${root_dev}'"' |  sed -nr 's/NAME="([^"]+)".*/\1/p')
  Info "Found storage devices $other_devices to use"
  echo "$other_devices"
}

# Wait for a device for certain time interval. If device is found 0 is
# returned otherwise 1.
wait_for_dev() {
  local devpath=$1
  local timeout=$DEVICE_WAIT_TIMEOUT

  if [ -b "$devpath" ];then
    Info "Device node $devpath exists."
    return 0
  fi

  if [ -z "$DEVICE_WAIT_TIMEOUT" ] || [ "$DEVICE_WAIT_TIMEOUT" == "0" ];then
    Info "Not waiting for device $devpath as DEVICE_WAIT_TIMEOUT=${DEVICE_WAIT_TIMEOUT}."
    return 0
  fi

  while [ $timeout -gt 0 ]; do
    Info "Waiting for device $devpath to be available. Wait time remaining is $timeout seconds"
    if [ $timeout -le 5 ];then
      sleep $timeout
    else
      sleep 5
    fi
    timeout=$((timeout-5))
    if [ -b "$devpath" ]; then
      Info "Device node $devpath exists."
      return 0
    fi
  done

  Info "Timed out waiting for device $devpath"
  return 1
}

systemd_escaped_filename () {
  local escaped_path filename path=$1
  escaped_path=$(echo ${path}|sed 's|-|\\x2d|g')
  filename=$(echo ${escaped_path}.mount|sed 's|/|-|g' | cut -b 2-)
  echo $filename
}

#
# In the past we created a systemd mount target file, we no longer
# use it, but if one pre-existed we still need to handle it.
#
remove_systemd_mount_target () {
  local mp=$1
  local filename=$(systemd_escaped_filename $mp)
  if [ -f /etc/systemd/system/$filename ]; then
      if [ -x /usr/bin/systemctl ];then
	  systemctl disable $filename >/dev/null 2>&1
	  systemctl stop $filename >/dev/null 2>&1
	  systemctl daemon-reload
      fi
    rm -f /etc/systemd/system/$filename >/dev/null 2>&1
  fi
}

check_wipe_block_dev_sig() {
  local bdev=$1
  local sig

  if ! sig=$(wipefs -p $bdev); then
    Fatal "Failed to check signatures on device $bdev"
  fi

  [ "$sig" == "" ] && return 0

  if [ "$WIPE_SIGNATURES" == "true" ];then
    Info "Wipe Signatures is set to true. Any signatures on $bdev will be wiped."
    if ! wipefs -a $bdev; then
      Fatal "Failed to wipe signatures on device $bdev"
    fi
    return 0
  fi

  while IFS=, read offset uuid label type; do
    [ "$offset" == "# offset" ] && continue
    Fatal "Found $type signature on device ${bdev} at offset ${offset}. Wipe signatures using wipefs or use WIPE_SIGNATURES=true and retry."
  done <<< "$sig"
}

determine_partition_type() {
  local dev="$1" size_bytes part_type

  if ! size_bytes=$(blockdev --getsize64 "$dev"); then
    Fatal "Failed to determine size of disk $dev"
  fi

  if [ $size_bytes -gt $_MAX_MBR_SIZE_BYTES ];then
    part_type="gpt"
  else
    part_type="dos"
  fi
  echo $part_type
}

create_partition_sfdisk(){
  local dev="$1" part_type="$2" size part_label
  # Use a single partition of a whole device
  # TODO:
  #   * Consider gpt, or unpartitioned volumes
  #   * Error handling when partition(s) already exist
  #   * Deal with loop/nbd device names. See growpart code
  if [ "$part_type" == "gpt" ];then
    # Linux LVM GUID for GPT. Taken from Wiki.
    part_label="E6D6D379-F507-44C2-A23C-238F2A3DF928"
    # Create as big a partition as possible.
    size=""
  else
    part_label="8e"
    size=$(( $( awk "\$4 ~ /"$( basename $dev )"/ { print \$3 }" /proc/partitions ) * 2 - 2048 ))
  fi
    cat <<EOF | sfdisk $dev
unit: sectors
label: $part_type
2048,${size},$part_label
EOF
}

create_partition_parted(){
  local dev="$1"
  local part_type="$2"

  if [ "$part_type" == "gpt" ];then
    parted $dev --script mklabel gpt mkpart "container-partition" 0% 100% set 1 lvm on
  else
    parted $dev --script mklabel msdos mkpart primary 0% 100% set 1 lvm on
  fi
}

create_partition() {
  local dev="$1" part part_type

  part_type=`determine_partition_type "$dev"`

  if [ -x "/usr/sbin/parted" ]; then
    create_partition_parted $dev "$part_type"
  else
    create_partition_sfdisk $dev "$part_type"
  fi

  # Sometimes on slow storage it takes a while for partition device to
  # become available. Wait for device node to show up.
  if ! udevadm settle;then
    Fatal "udevadm settle after partition creation failed. Exiting."
  fi

  part=$(dev_query_first_child $dev)

  if ! wait_for_dev ${part}; then
    Fatal "Partition device ${part} is not available"
  fi
}

dev_query_first_child() {
  lsblk -npl -o NAME "$1" | tail -n +2 | head -1
}

create_disk_partitions() {
  local devs="$1" part

  for dev in $devs; do
    # wipefs /dev/disk does not wipe any lvm signatures which might be
    # present on /dev/diskpart1. This signature will become visible to
    # lvm udev rules and will kickstart volume creation as soon as partion
    # is created and race with further partition commands like wipefs,
    # pvcreate etc. So zero out first few MB of disk in an attempt to
    # wipe any lvm signatures on first partition.
    #
    # By now we have ownership of disk and we have checked there are no
    # signatures on disk or signatures have been wiped. Dont care about
    # any signatures now on in the middle of disk.
    Info "Writing zeros to first 4MB of device $dev"
    if ! dd if=/dev/zero of=$dev bs=1M count=4; then
      Fatal "Failed to zero first 4MB of device $dev"
    fi

    create_partition $dev
    part=$(dev_query_first_child $dev)

    # It now seems unnecessary to do wipefs on partition given we already
    # zeroed out first 4MB. Only time it will be required if partition
    # starts beyong 4MB. Keep it for now.
    if ! wipefs -f -a ${part}; then
      Fatal "Failed to wipe signatures on device ${part}"
    fi
    pvcreate ${part}
    _PVS="$_PVS ${part}"
  done
}

create_extend_volume_group() {
  if [ -z "$_VG_EXISTS" ]; then
    vgcreate $VG $_PVS
    _VG_CREATED=1
    _VG_EXISTS=1
  else
    # TODO:
    #   * Error handling when PV is already part of a VG
    vgextend $VG $_PVS
  fi
}

# This returns the mountpoint of $1
extra_lv_mountpoint() {
  local mounts
  local vg=$1
  local lv_name=$2
  local mount_dir=$3
  mounts=$(findmnt -n -o TARGET --source /dev/$vg/$lv_name | grep "^$mount_dir")
  echo $mounts
}

mount_extra_volume() {
  local vg=$1
  local lv_name=$2
  local mount_dir=$3
  remove_systemd_mount_target $mount_dir
  mounts=$(extra_lv_mountpoint $vg $lv_name $mount_dir)
  if [ -z "$mounts" ]; then
      mount -t ext4 /dev/$vg/$lv_name $mount_dir
  fi
}

# Create a logical volume of size specified by first argument. Name of the
# volume is specified using second argument.
create_lv() {
  local data_size=$1
  local data_lv_name=$2

  # striping like this means the max volume group size = min(of each device) * count(devices)
  if [[ $data_size == *%* ]]; then
    lvcreate -y -l $data_size -n $data_lv_name --stripes $(echo "$DEVS" |wc -l) $VG || return 1
  else
    lvcreate -y -L $data_size -n $data_lv_name --stripes $(echo "$DEVS" |wc -l) $VG || return 1
  fi
  return 0
}

setup_extra_volume() {
  local lv_name=$1
  local mount_dir=$2
  local lv_size=$3

  if ! create_lv $lv_size $lv_name; then
    Fatal "Failed to create volume $lv_name of size ${lv_size}."
  fi

  if ! mkfs -t ext4 /dev/$VG/$lv_name > /dev/null; then
    Fatal "Failed to create filesystem on /dev/$VG/${lv_name}."
  fi

  if ! mount_extra_volume $VG $lv_name $mount_dir; then
    Fatal "Failed to mount volume ${lv_name} on ${mount_dir}"
  fi

  # setup right selinux label first time fs is created. Mount operation
  # changes the label of directory to reflect the label on root inode
  # of mounted fs.
  if ! restore_selinux_context $mount_dir; then
    return 1
  fi
}

restore_selinux_context() {
  local dir=$1

  if ! restorecon -R $dir; then
    Error "restorecon -R $dir failed."
    return 1
  fi
}

partition_disks_create_vg() {
  local dev_list

  # If there is no volume group specified or no root volume group, there is
  # nothing to do in terms of dealing with disks.
  if [[ -n "$DEVS" && -n "$VG" ]]; then
    _DEVS_RESOLVED="${DEVS}" # already resolved
    dev_list="$_DEVS_RESOLVED" # wipe all disks

    for dev in $dev_list; do
      check_wipe_block_dev_sig $dev
    done
    create_disk_partitions "$dev_list"
    create_extend_volume_group
  fi
}

function run_command_create() {
  Info "Doing new script mountain volume group creation"

  DEVS="$(find_non_root_devices)"
  if [[ -z "$DEVS" ]]; then
    # allow it to be run on instance without extra storage
    Info "no devices to use so skipping volume storage creation"
    exit 0
  fi

  if ! vg_exists "$VG"; then
    Info "Checking/creating volume group"
    partition_disks_create_vg
    vgdisplay
    echo ""

    Info "Creating logical volume"
    setup_extra_volume "$LV_NAME" "$LV_MOUNT_PATH" '100%VG'
  else
    Info "Volume group '$VG' already exists, skipping an actions"
  fi
  lvdisplay
  Info "Done"
}

# source library (borrowed from container-storage-setup)
source $(dirname $0)/libcss.sh

# make sure lvm is installed
yum install -y lvm2

if [[ "$CREATE_VG" == "true" ]]; then
  # ensure mount point exists
  mkdir -p ${LV_MOUNT_PATH}
  run_command_create
fi

libcss.sh

#!/bin/bash
# Library for common functions (copied from https://github.com/projectatomic/container-storage-setup/blob/master/libcss.sh)

# echo info messages on stdout
Info() {
  # stdout is used to pass back output from bash functions
  #  so we use stderr
  echo "INFO: ${1}" >&2
}

# echo warn messages on stderr
Warn() {
  echo "WARN: ${1}" >&2
}

# echo error messages on stderr
Error() {
  echo "ERROR: ${1}" >&2
}

# echo error on stderr and exit with error code 1
Fatal() {
  Error "${1}"
  exit 1
}

# Check if passed in vg exists. Returns 0 if volume group exists.
vg_exists() {
  local vg=$1

  for vg_name in $(vgs --noheadings -o vg_name); do
    if [ "$vg_name" == "$vg" ]; then
      return 0
    fi
  done
  return 1
}