ec2-create-ubuntu-ami.sh

ec2-create-ubuntu-ami.sh

#!/bin/bash
set -euo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Thanks:
# * https://github.com/alestic/alestic-git/blob/master/bin/alestic-git-build-ami for the overall approach and ec2 commands
# * https://github.com/kickstarter/build-ubuntu-ami/blob/master/data/user_data.sh.erb for the user script run in chroot
# * https://blog.tinned-software.net/mount-raw-image-of-entire-disc/ for how to mount the raw image with losetup


# Required IAM policy (replace "aws-cn" by "aws" if outside of China):
#
# {
#     "Version": "2012-10-17",
#     "Statement": [
#         {
#             "Effect": "Allow",
#             "Action": [
#                 "ec2:RegisterImage",
#                 "ec2:DescribeVolumes",
#                 "ec2:CreateSnapshot",
#                 "ec2:DescribeSnapshots",
#                 "ec2:CreateVolume"
#             ],
#             "Resource": "*"
#         },
#         {
#             "Effect": "Allow",
#             "Action": [
#                 "ec2:DetachVolume",
#                 "ec2:AttachVolume",
#                 "ec2:DeleteVolume"
#             ],
#             "Resource": "*",
#             "Condition": {
#                 "StringEquals": {
#                     "ec2:ResourceTag/WithRole": "amibuilder"
#                 }
#             }
#         },
#         {
#             "Effect": "Allow",
#             "Action": "ec2:CreateTags",
#             "Resource": "arn:aws-cn:ec2:*:*:volume/*",
#             "Condition": {
#                 "StringEquals": {
#                     "ec2:CreateAction": "CreateVolume"
#                 }
#             }
#         },
#         {
#             "Effect": "Allow",
#             "Action": "ec2:CreateTags",
#             "Resource": "arn:aws-cn:ec2:*:*:snapshot/*",
#             "Condition": {
#                 "StringEquals": {
#                     "ec2:CreateAction": "CreateSnapshot"
#                 }
#             }
#         }
#     ]
# }
#
# Also, the VM running this script should have a tag "WithRole=amibuilder"

# Required AWS CLI config file:
#
# [default]
# aws_access_key_id=
# aws_secret_access_key=



# To install AWS cli
# From https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
# 
# curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
# unzip awscliv2.zip
# sudo ./aws/install
# rm -rf aws awscliv2.zip 

# To install nvme
# apt-get install nvme-cli

# To install qemu-img
# apt-get install qemu-utils

for needcommand in curl aws jq lsblk qemu-img losetup; do
  command -v "$needcommand" >/dev/null 2>&1 || { echo >&2 "This script requires ${needcommand}"; exit 1; }
done

# Check if disks are "sdX"/"xvdX" or "nvmeXnY", require nvme-cli if needed
if [[ "$( lsblk --json | jq --raw-output .blockdevices[].name | head -n1 | cut -c 1-4 )" == "nvme" ]]; then
  command -v nvme >/dev/null 2>&1 || { echo >&2 "This script requires nvme (apt-get install nvme-cli)"; exit 1; }
fi

CODENAME=
AZ=
ADDSIZE=
MORERECENT=
KEEPBIN=
CHROOT_SCRIPT=
IMGNAME=
while [ $# -gt 0 ]; do
  case $1 in
    --codename)          CODENAME=$2; shift 2 ;;
    --az)                AZ=$2; shift 2 ;; # (optional) We need the AZ to use for creating the volume. It should match where this script is running. Auto detected through instance metadata if missing.
    --if-more-recent)    MORERECENT=$2; shift 2 ;; # (optional) YYYYMMDD format
    --keep-binaries)     KEEPBIN=1; shift ;;
    --run-script)        CHROOT_SCRIPT=$2; shift 2 ;; 
    --add-size)          ADDSIZE=$2; shift 2 ;; # (optional) integer + unit (e.g. M or G)
    --label)             IMGNAME=$2; shift 2 ;;
    *)                   echo "$0: Unrecognized option: $1" >&2; exit 1;
  esac
done

if [[ -z "$CODENAME" ]]; then
  echo "$0: Missing --codename (e.g. bionic)" >&2; exit 1;
fi
if [[ -z "$IMGNAME" ]]; then
  IMGNAME=Ubuntu
fi

if [[ -z "$AWS_SHARED_CREDENTIALS_FILE" ]]; then
  echo "$0: Missing env var AWS_SHARED_CREDENTIALS_FILE" >&2; exit 1;
fi

if [[ ! -z "${CHROOT_SCRIPT}" ]]; then
  if [[ ! -x "${CHROOT_SCRIPT}" ]]; then
    echo "$0: --run-script target does not exist or is not executable" >&2; exit 1;
  fi
fi

if [[ -z "$AZ" ]]; then
  # Try to auto-detect the AZ
  AZ=$( curl -L -Ss "http://169.254.169.254/latest/meta-data/placement/availability-zone" )
  if [[ -z "$AZ" ]]; then
    echo "$0: Missing --az (e.g. cn-north-1a) and failed to auto-detect it" >&2; exit 1;
  fi
fi

# Prepare temp directory
BUILDTIME=$( date +%s )
TMPDIR=$DIR/images/tmp-${BUILDTIME}
mkdir -p "${TMPDIR}"
# shellcheck disable=SC2064
trap "rm -rf '${TMPDIR}'" EXIT


if [[ "${AZ:0:2}" == "cn" ]]; then
  # Use China mirror
  MIRRORBASEAPI=https://mirrors.nju.edu.cn/ubuntu-cloud-images
  MIRRORBASEDL=http://mirrors.nju.edu.cn/ubuntu-cloud-images # We use http for faster download ; it's OK because we verify the file sha256sum for integrity
else
  MIRRORBASEAPI=https://cloud-images.ubuntu.com
  MIRRORBASEDL=https://cloud-images.ubuntu.com
fi

# Check what is the most recent Ubuntu release available for our codename
curl -L -Ss "${MIRRORBASEAPI}/query/${CODENAME}/server/released-dl.current.txt" | grep amd64 > "${TMPDIR}/release.txt"
RELEASEDATE=$( cat "${TMPDIR}/release.txt" | cut -f 4 )
if [[ ! -z "$MORERECENT" ]]; then 
  # We verify the release is more recent than what we already have
  DAYS=$((RELEASEDATE - MORERECENT))
  if [[ "$DAYS" -le 0 ]]; then
    exit 0
  fi
fi
if [[ -z "$RELEASEDATE" ]]; then
  echo "Unknown codename $CODENAME" >&2
  exit 1
fi

# If we are here, we need to get the base image
mkdir -p "$DIR/images"
OUTPUT=$DIR/images/ubuntu-${CODENAME}-${RELEASEDATE}.qcow2
DLURL=$( cat "${TMPDIR}/release.txt" | cut -f 6 | sed -e 's/tar\.gz$/img/' )
if [[ ! -f "$OUTPUT" ]]; then 
  # Need to download
  curl -L -Ss "${MIRRORBASEDL}/${DLURL}" -o "${OUTPUT}"
fi
CSURL=$( dirname "${MIRRORBASEAPI}/${DLURL}" )/SHA256SUMS
CSNAME=$( basename "${MIRRORBASEAPI}/${DLURL}" )
CHECKSUM_EXPECTED=$( curl -L -Ss "${CSURL}" | grep "${CSNAME}" | cut -f 1 -d ' ' )
CHECKSUM_GOTTEN=$( sha256sum "${OUTPUT}" | cut -f 1 -d ' ' )
if [[ "$CHECKSUM_EXPECTED" != "$CHECKSUM_GOTTEN" ]]; then
  echo "$0: Bad checksum on download!" >&2; exit 1;
fi

LABEL="${IMGNAME} ${CODENAME} ${RELEASEDATE} (build ${BUILDTIME})"

# Convert qcow2 image to raw
RAWOUTPUT=$TMPDIR/image.raw
qemu-img convert -O raw "$OUTPUT" "$RAWOUTPUT"

if [[ -z "$KEEPBIN" ]]; then
  # No need to keep the source image
  rm -f "$OUTPUT"
fi

# Add space to the image if requested
if [[ ! -z "${ADDSIZE}" ]]; then
  qemu-img resize -f raw "$RAWOUTPUT" "+${ADDSIZE}"
fi

# Mount the volume
devloop=$( losetup -f ) # e.g. /dev/loop0
losetup -f -P "$RAWOUTPUT"

MOUNTPOINT=/mount/image
mkdir -p "$MOUNTPOINT"
mount "${devloop}p1" "$MOUNTPOINT"
echo "[SIDE EFFECT] Mounted ${devloop}p1 under $MOUNTPOINT"

# Run additional commands in a chroot
if [[ ! -z "${CHROOT_SCRIPT}" ]]; then

  # Use all the space
  if [[ ! -z "${ADDSIZE}" ]]; then
    growpart "$devloop" 1
    resize2fs "${devloop}p1"
  fi
  
  # Allow network access from chroot environment
  if [[ -e "$MOUNTPOINT/etc/resolv.conf" ]] || [[ -L "$MOUNTPOINT/etc/resolv.conf" ]]; then
    mv $MOUNTPOINT/etc/resolv.conf $MOUNTPOINT/etc/resolv.conf.bak
  fi
  cat /etc/resolv.conf > $MOUNTPOINT/etc/resolv.conf

  # Extra mounts
  mount -t proc none $MOUNTPOINT/proc/
  mount -t sysfs none $MOUNTPOINT/sys/
  mount -o bind /dev $MOUNTPOINT/dev/

  # prevent daemons from starting during apt-get
  echo -e '#!/bin/sh\nexit 101' > $MOUNTPOINT/usr/sbin/policy-rc.d
  chmod 755 $MOUNTPOINT/usr/sbin/policy-rc.d

  # RUN CUSTOM USER SCRIPT
  cp "${CHROOT_SCRIPT}" $MOUNTPOINT/tmp/custom_user_script
  chroot $MOUNTPOINT /tmp/custom_user_script
  rm -f $MOUNTPOINT/tmp/custom_user_script

  # Unmount extra mountpoints
  for PT in dev proc sys; do
    umount "$MOUNTPOINT/$PT"
  done

  # Put resolv.conf symlink back in place
  rm -f  $MOUNTPOINT/etc/resolv.conf
  if [[ -e "$MOUNTPOINT/etc/resolv.conf.bak" ]] || [[ -L "$MOUNTPOINT/etc/resolv.conf.bak" ]]; then
    mv $MOUNTPOINT/etc/resolv.conf.bak $MOUNTPOINT/etc/resolv.conf
  fi

  # Clean up policy-rc.d
  rm -f $MOUNTPOINT/usr/sbin/policy-rc.d

fi

umount -l "$MOUNTPOINT"
rmdir "$MOUNTPOINT"

losetup -d "$devloop"

export AWS_DEFAULT_REGION=${AZ: : -1} # remove the last character to convert the AZ code into a region code

# List the volumes currently attached
lsblk --json | jq --raw-output .blockdevices[].name | sort > "${TMPDIR}/volumes-before.txt"
cp "${TMPDIR}/volumes-before.txt" "${TMPDIR}/volumes-after.txt"

# Create and attach a temporary EBS volume
SIZE=$( du --block-size=1G --apparent-size "$RAWOUTPUT" | cut -f 1 ) # --apparent-size is mandatory (with it 2.2 GB ; without 1.1 GB)
volumeid=$(aws ec2 create-volume --tag-specifications 'ResourceType=volume,Tags=[{Key=WithRole,Value=amibuilder},{Key=Name,Value="AMI building"}]' --availability-zone "$AZ" --size "$SIZE" --volume-type gp2 --output text --query 'VolumeId' )
echo "[SIDE EFFECT] Created volume $volumeid"
while aws ec2 describe-volumes --volume-id "$volumeid" --output text --query 'Volumes[*].State' | grep -v -q available; do
  sleep 3; 
done

instance_id=$(curl -L -Ss http://169.254.169.254/latest/meta-data/instance-id)
aws ec2 attach-volume --device /dev/sdi --instance-id "$instance_id" --volume-id "$volumeid" --output text --query 'State'
echo "[SIDE EFFECT] Attached volume $volumeid to instance $instance_id under device path /dev/sdi"
while cmp "${TMPDIR}/volumes-before.txt" "${TMPDIR}/volumes-after.txt" >/dev/null 2>/dev/null; do
  lsblk --json | jq --raw-output .blockdevices[].name | sort > "${TMPDIR}/volumes-after.txt"
  sleep 3;
done

# If we are here, it means a new device appeared
NEWDEV=$( comm -13 "${TMPDIR}/volumes-before.txt" "${TMPDIR}/volumes-after.txt" | head -n1 )
case "$NEWDEV" in
  sdi|xvdi) 
    dev=/dev/${NEWDEV}
    ;;
  nvm*) 
    # we can use nvme cli to verify it's the right volume
    dev=/dev/${NEWDEV}
    if [[ "$volumeid" != "$( nvme id-ctrl "$dev" --output-format=json | jq --raw-output .sn | sed -e 's/vol/vol-/' )" ]]; then
      echo >&2 "NVMe volume mismatch"
      exit 1
    fi
    ;;
  *)
    echo >&2 "Did not find the volume"
    exit 1
    ;;
esac

# Write the image to disk
dd if="$RAWOUTPUT" of="$dev" bs=8M
rm -f "$RAWOUTPUT"
 
# Detach the volume
aws ec2 detach-volume --volume-id "$volumeid" --output text --query 'State'
while aws ec2 describe-volumes --volume-id "$volumeid" --output text --query 'Volumes[*].State' | grep -v -q available; do
  sleep 3; 
done

# Create a snapshot of the volume
snapshotid=$(aws ec2 create-snapshot --tag-specifications 'ResourceType=snapshot,Tags=[{Key=Name,Value="For AMI"}]' --description "${LABEL}" --volume-id "$volumeid" --output text --query 'SnapshotId')
echo "[SIDE EFFECT] Created a snapshot $snapshotid"
while aws ec2 describe-snapshots --snapshot-id "$snapshotid" --output text --query 'Snapshots[*].State' | grep -q pending; do
  sleep 10;
done

# We can now delete the volume
aws ec2 delete-volume --volume-id "$volumeid" --output text

# Register the snapshot as a new AMI
block_device_mapping=$(cat <<EOF
[
  {
    "DeviceName": "/dev/sda1",
    "Ebs": {
      "DeleteOnTermination": false, 
      "SnapshotId": "$snapshotid",
      "VolumeSize": $SIZE,
      "VolumeType": "gp2"
    }
  }, {
    "DeviceName": "/dev/sdb",
    "VirtualName": "ephemeral0"
  }
]
EOF
)

amiid=$(aws ec2 register-image --name "${LABEL}" --ena-support --description "${LABEL}" --architecture x86_64 --virtualization-type hvm --block-device-mapping "$block_device_mapping" --root-device-name "/dev/sda1" --output text --query 'ImageId')
echo "[SIDE EFFECT] Created AMI ${amiid}"
echo "Published AMI ${amiid} in region ${AWS_DEFAULT_REGION}"

exit 0

sample-usage.sh

sudo env AWS_SHARED_CREDENTIALS_FILE=/home/ubuntu/.aws-creds \
bash /home/ubuntu/ec2-create-ubuntu-ami.sh \
--codename bionic --keep-binaries --add-size 1G \
--run-script /home/ubuntu/ubuntu-bionic-extra.sh \
--label ubuntu-docker-host 2>&1 | tee log-docker-ami-2020-10-12.log

ubuntu-bionic-extra.sh

#!/bin/bash

set -euo pipefail
set -vx

AZ=$( curl -L -Ss "http://169.254.169.254/latest/meta-data/placement/availability-zone" )

# Ubuntu mirror
if [[ "${AZ:0:2}" == "cn" ]]; then
  sed -i 's#://.*\.ubuntu\.com[^ ]*#://mirrors.tuna.tsinghua.edu.cn/ubuntu/#gi' /etc/apt/sources.list
fi

# Update but skipping kernel 
# From https://www.bonusbits.com/wiki/HowTo:Upgrade_Ubuntu_without_Updating_the_Kernel
apt-mark hold linux-image-generic linux-headers-generic
apt-get update
apt-get -y upgrade
apt-mark unhold linux-image-generic linux-headers-generic

# Add utilities
apt-get install --no-install-recommends -y apt-transport-https ca-certificates curl software-properties-common make zip unzip jq

# Install docker
if [[ "${AZ:0:2}" == "cn" ]]; then
  dockerkey=https://download.docker.com/linux/ubuntu/gpg
  dockerrepo=https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu/
else
  dockerkey=https://download.docker.com/linux/ubuntu/gpg
  dockerrepo=https://download.docker.com/linux/ubuntu
fi
curl -L -o /etc/apt/trusted.gpg.d/docker.asc ${dockerkey}
add-apt-repository "deb [arch=amd64] ${dockerrepo} $(lsb_release -cs) stable"
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io
systemctl enable docker

# Allow ubuntu to use docker without sudo
# NO! user ubuntu doesn't exist at this stage!
#usermod -aG docker ubuntu

# Remove stupid indentation rules for vim
rm -f /usr/share/vim/vim80/indent.vim

# Use vim as default editor
update-alternatives --set editor /usr/bin/vim.basic

# Install netdata
curl -L -o /etc/apt/trusted.gpg.d/netdata.asc https://packagecloud.io/netdata/netdata/gpgkey
add-apt-repository -s "deb https://packagecloud.io/netdata/netdata/ubuntu/ bionic main"
apt-get update
apt-get install -y netdata

# Install fluentbit
curl -L -o /etc/apt/trusted.gpg.d/fluentbit.asc http://packages.fluentbit.io/fluentbit.key
add-apt-repository "deb http://packages.fluentbit.io/ubuntu/bionic bionic main"
apt-get update
apt-get install -y td-agent-bit

exit 0