Ceph QEMU trimmer script

ceph-trimmer.sh

#!/bin/bash
# Since Ceph's TRIM/discard support for the rbd kernel module driver has been
# an open TODO for half a decade, this script attempts to boot up a minimal
# Ubuntu QEMU+KVM guest to do such bidding. Due to mysterious raisins, librbd
# has discard support, whereas the rbd kernel module does not... and QEMU uses
# librbd, hence this wasteful script.
#
# On "decent" server hardware at home, this completes in just under 60sec for
# a 100GB rbd image.
# 
# Basic process is as follows:
# - Generate the user-data and meta-data files to seed cloud-init
# - Generate the ISO image that cloud-init will see by checking volume names
#   and CD devices
# - Check for a base-d01.img file representing a base OS image
# - If not present, download it (Ubuntu 13.10 server, amd64)
# - Build a throwaway copy-on-write d01.img based on the original OS image
# - Boot from the throwaway image, and include:
#    - $QEMU_CORES SMP cores with KVM acceleration
#    - $QEMU_MEMORY of memory
#    - OS disk on a virtio bus (mostly since we can't have 3 devices in
#      QEMU's IDE bus)
#    - The rbd block device on a IDE bus, with QEMU discard options set
#    - The cd device with the generated cloud-init ISO loaded
#    - A NIC for host access if necessary
#    - Console output via -nographic, which can be captured/saved
# - Cloud-init boots and configures the machine
# - Cloud-init mounts the RBD block device, begins an fstrim run, as well as
#   an xfs_repair for good measure
# - Unmount the drive and poweroff the guest only if umount was a wild success
# - Throw away the... throwaway image
# - Back away slowly
#
# Thankfully, cloud-init sends the output of all this to the console, so we can
# archive it somewhere, perhaps even snapshotting the RBD image upon successful
# runs.


## Something go wrong? HOLD THE PHONE.
set -o errexit

GUEST_RELEASE=trusty

RBD_POOL=media
RBD_IMAGE=media01

QEMU_MEMORY=2048
QEMU_CORES=2
QEMU_HOSTNAME="ceph-trimmer"
QEMU_DOMAIN="ads1.sysnw.net"
QEMU_INST_ID="i-$(date +%Y%m%d%H%M%S)"

XFS_OPTS="noatime,nodiratime,nobarrier,discard"
XFS_DISCARD_GRANULARITY=512

HOST_SSH_PORT=2222


#############################################################################
## Make sure we've got everything we need on the host (really, this is it)
export DEBIAN_FRONTEND=noninteractive
export DEBCONF_NONINTERACTIVE_SEEN=true
apt-get install -q -y curl \
                      genisoimage \
                      qemu-kvm \
                      qemu-utils


#############################################################################
## Create the instance user metadata file
cat > user-data <<EOF
#cloud-config
hostname: $QEMU_HOSTNAME
fqdn: $QEMU_HOSTNAME.$QEMU_DOMAIN
manage_etc_hosts: True
timezone: US/Central
locale: en_US.UTF-8

apt_mirror: http://mirror01.ads1.sysnw.net/ubuntu/
apt_preserve_sources_list: false
apt_update: true
package_update: false
packages:
 - xfsprogs

ssh_authorized_keys:
 - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAtwEDMK45A7HJ5l1PdZ02S1ZSKmCn2C6+z+BFDHTcdGthdbATWDB0QNP+xHdlXfrSj+mJwMWZT9+VydNhWWdaJOAVEl9GQncHLzJZ6DhRbT78Qp9tcCq6I8kfIktmCiQ5N1T6Zx/PcxeXJa46BscV+cveBlN8D7fJyWVLEKN4QZx+BDNCmshD4XX4qQUGEAWClP7sv1ZUwUQilNxvcT9OfvN9tA1vZeIXFffj0rkV3+h+SloPqSp5ckw2hJF444kwcuthyEzF83DMKIyUlExp7+ALaf8za7OgpNqYITO+ZQi5y9VX1+RQzHpeWpED2lVpYWdNT1+03GfQcBB6SbtjqQ== testkey.qemu-trimmer

runcmd:
 - mkdir -p /mnt/rbd
 - mount -t xfs -o $XFS_OPTS /dev/sda /mnt/rbd
 - sh -c 'df -h && fstrim -v /mnt/rbd && umount /mnt/rbd && xfs_repair /dev/sda'

power_state:
 mode: poweroff
 timeout: 30

EOF


#############################################################################
## Create the instance meta data file
cat > meta-data <<EOF
instance-id: $QEMU_INST_ID
local-hostname: $QEMU_HOSTNAME
EOF


#############################################################################
## Build nocloud config drive ISO image
genisoimage -output d02-nocloud-seed.iso -volid cidata -joliet -rock user-data meta-data


#############################################################################
## If we don't have a copy of the base OS already, grab it
if [ ! -f base-d01.img ]; then
    curl -o base-d01.img http://cloud-images.ubuntu.com/${GUEST_RELEASE}/current/${GUEST_RELEASE}-server-cloudimg-amd64-disk1.img
fi


#############################################################################
## Start with a clean OS disk each time
[ -f d01.img ] && rm -f d01.img
qemu-img create -b base-d01.img -f qcow2 d01.img 10G


#############################################################################
## Fire up and trim the fat away (in seconds!)
time \
qemu-system-x86_64 \
    -machine accel=kvm:tcg \
    -cpu host \
    -smp $QEMU_CORES \
    -m $QEMU_MEMORY \
    -drive id=d01,if=none,format=qcow2,file=d01.img \
    -drive id=d02,if=none,format=raw,file=rbd:$RBD_POOL/$RBD_IMAGE, \
    -drive id=c01,if=none,format=raw,file=d02-nocloud-seed.iso,media=cdrom \
    -device driver=virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=d01,id=vd01,bootindex=1 \
    -device driver=ide-hd,bus=ide.1,unit=0,drive=d02,discard_granularity=$XFS_DISCARD_GRANULARITY \
    -device driver=ide-cd,bus=ide.1,unit=1,drive=c01 \
    -net nic \
    -net user,hostfwd=tcp::$HOST_SSH_PORT-:22 \
    -nographic


#############################################################################
## Clean up after yourself, bro
rm -f d01.img \
      d02-nocloud-seed.iso \
      user-data \
      meta-data