Skip to content

Instantly share code, notes, and snippets.

@matrey
Created October 15, 2020 05:41
Show Gist options
  • Save matrey/66d697ef540f0da8933a341524ea9fd7 to your computer and use it in GitHub Desktop.
Save matrey/66d697ef540f0da8933a341524ea9fd7 to your computer and use it in GitHub Desktop.
ec2-create-ubuntu-ami.sh
#!/bin/bash
set -euo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Thanks:
# * https://github.com/alestic/alestic-git/blob/master/bin/alestic-git-build-ami for the overall approach and ec2 commands
# * https://github.com/kickstarter/build-ubuntu-ami/blob/master/data/user_data.sh.erb for the user script run in chroot
# * https://blog.tinned-software.net/mount-raw-image-of-entire-disc/ for how to mount the raw image with losetup
# Required IAM policy (replace "aws-cn" by "aws" if outside of China):
#
# {
# "Version": "2012-10-17",
# "Statement": [
# {
# "Effect": "Allow",
# "Action": [
# "ec2:RegisterImage",
# "ec2:DescribeVolumes",
# "ec2:CreateSnapshot",
# "ec2:DescribeSnapshots",
# "ec2:CreateVolume"
# ],
# "Resource": "*"
# },
# {
# "Effect": "Allow",
# "Action": [
# "ec2:DetachVolume",
# "ec2:AttachVolume",
# "ec2:DeleteVolume"
# ],
# "Resource": "*",
# "Condition": {
# "StringEquals": {
# "ec2:ResourceTag/WithRole": "amibuilder"
# }
# }
# },
# {
# "Effect": "Allow",
# "Action": "ec2:CreateTags",
# "Resource": "arn:aws-cn:ec2:*:*:volume/*",
# "Condition": {
# "StringEquals": {
# "ec2:CreateAction": "CreateVolume"
# }
# }
# },
# {
# "Effect": "Allow",
# "Action": "ec2:CreateTags",
# "Resource": "arn:aws-cn:ec2:*:*:snapshot/*",
# "Condition": {
# "StringEquals": {
# "ec2:CreateAction": "CreateSnapshot"
# }
# }
# }
# ]
# }
#
# Also, the VM running this script should have a tag "WithRole=amibuilder"
# Required AWS CLI config file:
#
# [default]
# aws_access_key_id=
# aws_secret_access_key=
# To install AWS cli
# From https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
#
# curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
# unzip awscliv2.zip
# sudo ./aws/install
# rm -rf aws awscliv2.zip
# To install nvme
# apt-get install nvme-cli
# To install qemu-img
# apt-get install qemu-utils
for needcommand in curl aws jq lsblk qemu-img losetup; do
command -v "$needcommand" >/dev/null 2>&1 || { echo >&2 "This script requires ${needcommand}"; exit 1; }
done
# Check if disks are "sdX"/"xvdX" or "nvmeXnY", require nvme-cli if needed
if [[ "$( lsblk --json | jq --raw-output .blockdevices[].name | head -n1 | cut -c 1-4 )" == "nvme" ]]; then
command -v nvme >/dev/null 2>&1 || { echo >&2 "This script requires nvme (apt-get install nvme-cli)"; exit 1; }
fi
CODENAME=
AZ=
ADDSIZE=
MORERECENT=
KEEPBIN=
CHROOT_SCRIPT=
IMGNAME=
while [ $# -gt 0 ]; do
case $1 in
--codename) CODENAME=$2; shift 2 ;;
--az) AZ=$2; shift 2 ;; # (optional) We need the AZ to use for creating the volume. It should match where this script is running. Auto detected through instance metadata if missing.
--if-more-recent) MORERECENT=$2; shift 2 ;; # (optional) YYYYMMDD format
--keep-binaries) KEEPBIN=1; shift ;;
--run-script) CHROOT_SCRIPT=$2; shift 2 ;;
--add-size) ADDSIZE=$2; shift 2 ;; # (optional) integer + unit (e.g. M or G)
--label) IMGNAME=$2; shift 2 ;;
*) echo "$0: Unrecognized option: $1" >&2; exit 1;
esac
done
if [[ -z "$CODENAME" ]]; then
echo "$0: Missing --codename (e.g. bionic)" >&2; exit 1;
fi
if [[ -z "$IMGNAME" ]]; then
IMGNAME=Ubuntu
fi
if [[ -z "$AWS_SHARED_CREDENTIALS_FILE" ]]; then
echo "$0: Missing env var AWS_SHARED_CREDENTIALS_FILE" >&2; exit 1;
fi
if [[ ! -z "${CHROOT_SCRIPT}" ]]; then
if [[ ! -x "${CHROOT_SCRIPT}" ]]; then
echo "$0: --run-script target does not exist or is not executable" >&2; exit 1;
fi
fi
if [[ -z "$AZ" ]]; then
# Try to auto-detect the AZ
AZ=$( curl -L -Ss "http://169.254.169.254/latest/meta-data/placement/availability-zone" )
if [[ -z "$AZ" ]]; then
echo "$0: Missing --az (e.g. cn-north-1a) and failed to auto-detect it" >&2; exit 1;
fi
fi
# Prepare temp directory
BUILDTIME=$( date +%s )
TMPDIR=$DIR/images/tmp-${BUILDTIME}
mkdir -p "${TMPDIR}"
# shellcheck disable=SC2064
trap "rm -rf '${TMPDIR}'" EXIT
if [[ "${AZ:0:2}" == "cn" ]]; then
# Use China mirror
MIRRORBASEAPI=https://mirrors.nju.edu.cn/ubuntu-cloud-images
MIRRORBASEDL=http://mirrors.nju.edu.cn/ubuntu-cloud-images # We use http for faster download ; it's OK because we verify the file sha256sum for integrity
else
MIRRORBASEAPI=https://cloud-images.ubuntu.com
MIRRORBASEDL=https://cloud-images.ubuntu.com
fi
# Check what is the most recent Ubuntu release available for our codename
curl -L -Ss "${MIRRORBASEAPI}/query/${CODENAME}/server/released-dl.current.txt" | grep amd64 > "${TMPDIR}/release.txt"
RELEASEDATE=$( cat "${TMPDIR}/release.txt" | cut -f 4 )
if [[ ! -z "$MORERECENT" ]]; then
# We verify the release is more recent than what we already have
DAYS=$((RELEASEDATE - MORERECENT))
if [[ "$DAYS" -le 0 ]]; then
exit 0
fi
fi
if [[ -z "$RELEASEDATE" ]]; then
echo "Unknown codename $CODENAME" >&2
exit 1
fi
# If we are here, we need to get the base image
mkdir -p "$DIR/images"
OUTPUT=$DIR/images/ubuntu-${CODENAME}-${RELEASEDATE}.qcow2
DLURL=$( cat "${TMPDIR}/release.txt" | cut -f 6 | sed -e 's/tar\.gz$/img/' )
if [[ ! -f "$OUTPUT" ]]; then
# Need to download
curl -L -Ss "${MIRRORBASEDL}/${DLURL}" -o "${OUTPUT}"
fi
CSURL=$( dirname "${MIRRORBASEAPI}/${DLURL}" )/SHA256SUMS
CSNAME=$( basename "${MIRRORBASEAPI}/${DLURL}" )
CHECKSUM_EXPECTED=$( curl -L -Ss "${CSURL}" | grep "${CSNAME}" | cut -f 1 -d ' ' )
CHECKSUM_GOTTEN=$( sha256sum "${OUTPUT}" | cut -f 1 -d ' ' )
if [[ "$CHECKSUM_EXPECTED" != "$CHECKSUM_GOTTEN" ]]; then
echo "$0: Bad checksum on download!" >&2; exit 1;
fi
LABEL="${IMGNAME} ${CODENAME} ${RELEASEDATE} (build ${BUILDTIME})"
# Convert qcow2 image to raw
RAWOUTPUT=$TMPDIR/image.raw
qemu-img convert -O raw "$OUTPUT" "$RAWOUTPUT"
if [[ -z "$KEEPBIN" ]]; then
# No need to keep the source image
rm -f "$OUTPUT"
fi
# Add space to the image if requested
if [[ ! -z "${ADDSIZE}" ]]; then
qemu-img resize -f raw "$RAWOUTPUT" "+${ADDSIZE}"
fi
# Mount the volume
devloop=$( losetup -f ) # e.g. /dev/loop0
losetup -f -P "$RAWOUTPUT"
MOUNTPOINT=/mount/image
mkdir -p "$MOUNTPOINT"
mount "${devloop}p1" "$MOUNTPOINT"
echo "[SIDE EFFECT] Mounted ${devloop}p1 under $MOUNTPOINT"
# Run additional commands in a chroot
if [[ ! -z "${CHROOT_SCRIPT}" ]]; then
# Use all the space
if [[ ! -z "${ADDSIZE}" ]]; then
growpart "$devloop" 1
resize2fs "${devloop}p1"
fi
# Allow network access from chroot environment
if [[ -e "$MOUNTPOINT/etc/resolv.conf" ]] || [[ -L "$MOUNTPOINT/etc/resolv.conf" ]]; then
mv $MOUNTPOINT/etc/resolv.conf $MOUNTPOINT/etc/resolv.conf.bak
fi
cat /etc/resolv.conf > $MOUNTPOINT/etc/resolv.conf
# Extra mounts
mount -t proc none $MOUNTPOINT/proc/
mount -t sysfs none $MOUNTPOINT/sys/
mount -o bind /dev $MOUNTPOINT/dev/
# prevent daemons from starting during apt-get
echo -e '#!/bin/sh\nexit 101' > $MOUNTPOINT/usr/sbin/policy-rc.d
chmod 755 $MOUNTPOINT/usr/sbin/policy-rc.d
# RUN CUSTOM USER SCRIPT
cp "${CHROOT_SCRIPT}" $MOUNTPOINT/tmp/custom_user_script
chroot $MOUNTPOINT /tmp/custom_user_script
rm -f $MOUNTPOINT/tmp/custom_user_script
# Unmount extra mountpoints
for PT in dev proc sys; do
umount "$MOUNTPOINT/$PT"
done
# Put resolv.conf symlink back in place
rm -f $MOUNTPOINT/etc/resolv.conf
if [[ -e "$MOUNTPOINT/etc/resolv.conf.bak" ]] || [[ -L "$MOUNTPOINT/etc/resolv.conf.bak" ]]; then
mv $MOUNTPOINT/etc/resolv.conf.bak $MOUNTPOINT/etc/resolv.conf
fi
# Clean up policy-rc.d
rm -f $MOUNTPOINT/usr/sbin/policy-rc.d
fi
umount -l "$MOUNTPOINT"
rmdir "$MOUNTPOINT"
losetup -d "$devloop"
export AWS_DEFAULT_REGION=${AZ: : -1} # remove the last character to convert the AZ code into a region code
# List the volumes currently attached
lsblk --json | jq --raw-output .blockdevices[].name | sort > "${TMPDIR}/volumes-before.txt"
cp "${TMPDIR}/volumes-before.txt" "${TMPDIR}/volumes-after.txt"
# Create and attach a temporary EBS volume
SIZE=$( du --block-size=1G --apparent-size "$RAWOUTPUT" | cut -f 1 ) # --apparent-size is mandatory (with it 2.2 GB ; without 1.1 GB)
volumeid=$(aws ec2 create-volume --tag-specifications 'ResourceType=volume,Tags=[{Key=WithRole,Value=amibuilder},{Key=Name,Value="AMI building"}]' --availability-zone "$AZ" --size "$SIZE" --volume-type gp2 --output text --query 'VolumeId' )
echo "[SIDE EFFECT] Created volume $volumeid"
while aws ec2 describe-volumes --volume-id "$volumeid" --output text --query 'Volumes[*].State' | grep -v -q available; do
sleep 3;
done
instance_id=$(curl -L -Ss http://169.254.169.254/latest/meta-data/instance-id)
aws ec2 attach-volume --device /dev/sdi --instance-id "$instance_id" --volume-id "$volumeid" --output text --query 'State'
echo "[SIDE EFFECT] Attached volume $volumeid to instance $instance_id under device path /dev/sdi"
while cmp "${TMPDIR}/volumes-before.txt" "${TMPDIR}/volumes-after.txt" >/dev/null 2>/dev/null; do
lsblk --json | jq --raw-output .blockdevices[].name | sort > "${TMPDIR}/volumes-after.txt"
sleep 3;
done
# If we are here, it means a new device appeared
NEWDEV=$( comm -13 "${TMPDIR}/volumes-before.txt" "${TMPDIR}/volumes-after.txt" | head -n1 )
case "$NEWDEV" in
sdi|xvdi)
dev=/dev/${NEWDEV}
;;
nvm*)
# we can use nvme cli to verify it's the right volume
dev=/dev/${NEWDEV}
if [[ "$volumeid" != "$( nvme id-ctrl "$dev" --output-format=json | jq --raw-output .sn | sed -e 's/vol/vol-/' )" ]]; then
echo >&2 "NVMe volume mismatch"
exit 1
fi
;;
*)
echo >&2 "Did not find the volume"
exit 1
;;
esac
# Write the image to disk
dd if="$RAWOUTPUT" of="$dev" bs=8M
rm -f "$RAWOUTPUT"
# Detach the volume
aws ec2 detach-volume --volume-id "$volumeid" --output text --query 'State'
while aws ec2 describe-volumes --volume-id "$volumeid" --output text --query 'Volumes[*].State' | grep -v -q available; do
sleep 3;
done
# Create a snapshot of the volume
snapshotid=$(aws ec2 create-snapshot --tag-specifications 'ResourceType=snapshot,Tags=[{Key=Name,Value="For AMI"}]' --description "${LABEL}" --volume-id "$volumeid" --output text --query 'SnapshotId')
echo "[SIDE EFFECT] Created a snapshot $snapshotid"
while aws ec2 describe-snapshots --snapshot-id "$snapshotid" --output text --query 'Snapshots[*].State' | grep -q pending; do
sleep 10;
done
# We can now delete the volume
aws ec2 delete-volume --volume-id "$volumeid" --output text
# Register the snapshot as a new AMI
block_device_mapping=$(cat <<EOF
[
{
"DeviceName": "/dev/sda1",
"Ebs": {
"DeleteOnTermination": false,
"SnapshotId": "$snapshotid",
"VolumeSize": $SIZE,
"VolumeType": "gp2"
}
}, {
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
}
]
EOF
)
amiid=$(aws ec2 register-image --name "${LABEL}" --ena-support --description "${LABEL}" --architecture x86_64 --virtualization-type hvm --block-device-mapping "$block_device_mapping" --root-device-name "/dev/sda1" --output text --query 'ImageId')
echo "[SIDE EFFECT] Created AMI ${amiid}"
echo "Published AMI ${amiid} in region ${AWS_DEFAULT_REGION}"
exit 0
sudo env AWS_SHARED_CREDENTIALS_FILE=/home/ubuntu/.aws-creds \
bash /home/ubuntu/ec2-create-ubuntu-ami.sh \
--codename bionic --keep-binaries --add-size 1G \
--run-script /home/ubuntu/ubuntu-bionic-extra.sh \
--label ubuntu-docker-host 2>&1 | tee log-docker-ami-2020-10-12.log
#!/bin/bash
set -euo pipefail
set -vx
AZ=$( curl -L -Ss "http://169.254.169.254/latest/meta-data/placement/availability-zone" )
# Ubuntu mirror
if [[ "${AZ:0:2}" == "cn" ]]; then
sed -i 's#://.*\.ubuntu\.com[^ ]*#://mirrors.tuna.tsinghua.edu.cn/ubuntu/#gi' /etc/apt/sources.list
fi
# Update but skipping kernel
# From https://www.bonusbits.com/wiki/HowTo:Upgrade_Ubuntu_without_Updating_the_Kernel
apt-mark hold linux-image-generic linux-headers-generic
apt-get update
apt-get -y upgrade
apt-mark unhold linux-image-generic linux-headers-generic
# Add utilities
apt-get install --no-install-recommends -y apt-transport-https ca-certificates curl software-properties-common make zip unzip jq
# Install docker
if [[ "${AZ:0:2}" == "cn" ]]; then
dockerkey=https://download.docker.com/linux/ubuntu/gpg
dockerrepo=https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu/
else
dockerkey=https://download.docker.com/linux/ubuntu/gpg
dockerrepo=https://download.docker.com/linux/ubuntu
fi
curl -L -o /etc/apt/trusted.gpg.d/docker.asc ${dockerkey}
add-apt-repository "deb [arch=amd64] ${dockerrepo} $(lsb_release -cs) stable"
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io
systemctl enable docker
# Allow ubuntu to use docker without sudo
# NO! user ubuntu doesn't exist at this stage!
#usermod -aG docker ubuntu
# Remove stupid indentation rules for vim
rm -f /usr/share/vim/vim80/indent.vim
# Use vim as default editor
update-alternatives --set editor /usr/bin/vim.basic
# Install netdata
curl -L -o /etc/apt/trusted.gpg.d/netdata.asc https://packagecloud.io/netdata/netdata/gpgkey
add-apt-repository -s "deb https://packagecloud.io/netdata/netdata/ubuntu/ bionic main"
apt-get update
apt-get install -y netdata
# Install fluentbit
curl -L -o /etc/apt/trusted.gpg.d/fluentbit.asc http://packages.fluentbit.io/fluentbit.key
add-apt-repository "deb http://packages.fluentbit.io/ubuntu/bionic bionic main"
apt-get update
apt-get install -y td-agent-bit
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment