docker_to_nvme_ephemeral.sh

#!/bin/bash
#
# This script will attempt to detect any ephemeral drives on an EC2 node and create a RAID-0 stripe
# mounted at /mnt. It should be run early on the first boot of the system.
set -exo pipefail

METADATA_URL_BASE="http://169.254.169.254/2012-01-12"

# Configure Raid - take into account xvdb or sdb
root_drive=`df -h | grep -v grep | awk 'NR==4{print $1}'`

if [ "$root_drive" == "/dev/xvda1" ]; then
  echo "Detected 'xvd' drive naming scheme (root: $root_drive)"
  DRIVE_SCHEME='xvd'
elif [ "$root_drive" == "/dev/nvme1n1p1" ]; then
  echo "Detected nvme device"
  DRIVE_SCHEME='nvme'
else
  echo "Detected 'sd' drive naming scheme (root: $root_drive)"
  DRIVE_SCHEME='sd'
fi

# figure out how many ephemerals we have by querying the metadata API, and then:
#  - convert the drive name returned from the API to the hosts DRIVE_SCHEME, if necessary
#  - verify a matching device is available in /dev/
drives=""
ephemeral_count=0
ephemerals=$(fdisk -l | grep Disk | grep 'nvme' | awk '{print $2 }' | sed 's/://')
for e in $ephemerals; do
  echo "Probing $e .."
  device_name="$e"

  device_path="$device_name"

  # test that the device actually exists since you can request more ephemeral drives than are available
  # for an instance type and the meta-data API will happily tell you it exists when it really does not.
  if [ -b $device_path ]; then
    echo "Detected ephemeral disk: $device_path"
    root_comp=$(echo $root_drive | sed 's/p1//')
    if [ "$root_comp" != "$device_path" ]; then
      drives="$drives $device_path"
      ephemeral_count=$((ephemeral_count + 1 ))
    fi
  else
    echo "Ephemeral disk $e, $device_path is not present. skipping"
  fi
done

if [ "$ephemeral_count" = 0 ]; then
  echo "No ephemeral disk detected. exiting"
  exit 0
fi

echo "ephemeral count $ephemeral_count"

umount /mnt || true

# Overwrite first few blocks in case there is a filesystem, otherwise mdadm will prompt for input
for drive in $drives; do
  dd if=/dev/zero of=$drive bs=4096 count=1024
done

echo $ephemeral_count
echo $drives

# stripe devices, format fs and mount
mdadm --create /dev/md0 --force --raid-devices=$ephemeral_count --level=0 $drives
mkfs.ext4 /dev/md0
mount -t ext4 -o rw,async,noatime,comment=md0 /dev/md0 /mnt

# Remove xvdb/sdb from fstab
chmod 777 /etc/fstab
sed -i.bak "/${DRIVE_SCHEME}b/d" /etc/fstab

# Make raid appear on reboot
echo "/dev/md0 /mnt ext4 rw,async,noatime,comment=md0 0 2" | tee -a /etc/fstab
chmod a+rwx /mnt
sleep 10

mkdir -p /mnt/docker

cat << EOF > /etc/docker/daemon.json
{
  "metrics-addr" : "127.0.0.1:4999",
  "data-root": "/mnt/docker",
  "storage-driver": "overlay2",
  "default-shm-size":"1G",
  "log-driver": "json-file",
  "log-opts": {
	"max-size": "2m",
	"max-file":"3",
	"labels": "docker",
	"env": "os,customer"
  },
  "default-ulimits": {
    "nofile": {
      "Name": "nofile",
      "Hard": 120000,
      "Soft": 64000
    }
  },
  "mtu": 0,
  "max-concurrent-downloads": 3,
  "max-concurrent-uploads": 5,
  "ipv6": true,
  "fixed-cidr-v6": "2001:db8:1::/64",
  "iptables": true,
  "ip-forward": true,
  "experimental" : true,
  "features": { "buildkit": true}
}
EOF

systemctl restart docker.service