Upstart Job to Manage EC2 Ephemeral Devices

manage-ephemeral.conf

# manage-ephemeral - manages our ephemeral devices, if present
#
# Installation:
# /usr/bin/sudo /bin/cp /path/to/manage-ephemeral.conf /etc/init/manage-ephemeral.conf
# /usr/bin/sudo /bin/chmod 644 /etc/init/manage-ephemeral.conf
# /usr/bin/sudo /bin/ln /lib/init/upstart-job /etc/init.d/manage-ephemeral

description     "Manage Ephemeral Devices"
author          "Bubba Hines <rob@stechstudio.com>"

start on startup

exec /path/to/manageEphemeral.sh

manageEphemeral.sh

#!/bin/bash
## Usage: manageEphemeral.sh
##
##
## Ensure you have installed the tools required:
##  sudo apt-get -y install rpcbind nfs-common &>/dev/null
##

###################
##
## Utility Function to handle logging for us.
##
###################
function log {
    if ${LOG}; then
        echo [`/bin/date`] $1
    fi
}

##########################################################################
##
##                       Configuration
##
##########################################################################

# Logging. Set to true for debugging and testing; false when everything works. Logs 
# are written to /var/log/upstart/manage-ephemeral.log.
LOG=true

# We will have a directory to mount the data partition on
EPHEMERAL_DATA_DIR=/vol0

# User to own data
data_user="root"

# Group to own data
data_group="root"

# We might have up to two Ephemeral Devices
EPHEMERAL0_DEV=/dev/xvdb
EPHEMERAL1_DEV=/dev/xvdc

# Which we might make into a raid device
RAID0_DEV=/dev/md0

# Do we need a raid device? Assume we do not.
NEEDRAID=false

# Do we need to partion/format the device? Assume we do.
NEEDPARTITIONING=true

# Do we have an ephemeral device? Assume we do not.
HAVEEPHEMERAL=false

# What will the base device be? Assuming one ephemeral device
# it will default to just the first. If there are more, it will
# be set to the raid device.
BASEDEVICE=$EPHEMERAL0_DEV

# Self configure for raid device, base device, and whether we 
# even have ephemeral. If we have both devices:
if [ -e $EPHEMERAL0_DEV ] && [ -e $EPHEMERAL1_DEV ]; then
    log "We have Ephemeral and need to setup Raid"
    NEEDRAID=true
    HAVEEPHEMERAL=true
    BASEDEVICE=$RAID0_DEV
# If we only have one device:
elif [ -e $EPHEMERAL0_DEV ] ; then
    log "We have Ephemeral, but no need to setup Raid"
    NEEDRAID=false
    HAVEEPHEMERAL=true
    BASEDEVICE=$EPHEMERAL0_DEV
# We don't need anything
# And we can exit now with an error
else
    log "ERROR: No ephemeral storage found to work with"
    NEEDRAID=false
    HAVEEPHEMERAL=false
    exit 1
fi

# The partitions we intend to create
SWAP_PART=${BASEDEVICE}1
DATA_PART=${BASEDEVICE}2

# Unless we are doing the raid, in which case
# The partition names change a bit.
if [ "$NEEDRAID" == true ]; then
    SWAP_PART=${BASEDEVICE}p1
    DATA_PART=${BASEDEVICE}p2
fi

# Check the base device to see if it is already correctly partitioned
# and formated
if [ -e $BASEDEVICE ]; then
    SWAPPARTITIONEXISTS=`sudo parted $BASEDEVICE print|sed -n 7p|grep -c linux-swap`
    DATAPARTITIONEXISTS=`sudo parted $BASEDEVICE print|sed -n 8p|grep -c ext4`
    if [ "$SWAPPARTITIONEXISTS" -eq 1 ] && [ "$DATAPARTITIONEXISTS" -eq 1 ]; then
        log "No Need to Partition"
        NEEDPARTITIONING=false
    fi
fi

# And an ephemeral log directory
EPHEMERAL_LOG_BASE_DIR="$EPHEMERAL_DATA_DIR/log"
EPHEMERAL_LOG_DIR="$EPHEMERAL_LOG_BASE_DIR"

# And an ephemeral temp directory
EPHEMERAL_TMP_DIR="$EPHEMERAL_DATA_DIR/tmp"

## End of configuration
##########################################################################

##########################################################################
##
##                       FUNCTIONS
##
##########################################################################

####
## The primary program logic
####
function main {
    if [ "$NEEDPARTITIONING" == true ]; then
        log "We need to partition $BASEDEVICE"
        unmountEverything
        if [ "$NEEDRAID" == true ]; then
            log "We have 2 ephemeral drives, we need raid"
            createRaid
        fi
        partitionBaseDevice
        setupSwap
        formatData
        setupData
    else
        log "Looks like no need to partition; just setting up swap and data"
        # If the swap partition is mounted, get off it.
        swapoff $SWAP_PART > /dev/null 2>&1 || true
        swapon $SWAP_PART
        setupData
    fi
    log "Ephemeral is configured!"
}

####
## Ought to successfully unmount everything
####
function unmountEverything {
    log "Unmounting Evertyhing"
    # If the swap partition is mounted, get off it.
    swapoff $SWAP_PART > /dev/null 2>&1 || true

    # If the data partition is mounted, get off it.
    umount -l $DATA_PART > /dev/null 2>&1 || true

    # Does the md127 device exist? It is a thorn in my side
    # Apparently mdadm can create it for little or no reason
    # And jack up everything for me. Stupid mdadm.
    BADRAID_DEV=/dev/md127
    if [ -e "$BADRAID_DEV" ]; then
      log "Dissemble the $BADRAID_DEV Raid Device"
      # Stop the array
      mdadm --stop $BADRAID_DEV > /dev/null 2>&1 || true

      # If the raid device itself is already mounted, get off it.
      umount -l $BADRAID_DEV > /dev/null 2>&1 || true

      mdadm --remove $BADRAID_DEV > /dev/null 2>&1 || true

      #zero the superblock FOR EACH drive
      mdadm --zero-superblock $EPHEMERAL0_DEV > /dev/null 2>&1 || true
      mdadm --zero-superblock $EPHEMERAL1_DEV > /dev/null 2>&1 || true

      # If the first second device itself is already mounted, get off it.
      # After breaking down the raid device
      umount -l $EPHEMERAL1_DEV > /dev/null 2>&1 || true
    fi

    # Does the raid device exist?
    if [ -e "$RAID0_DEV" ]; then
      log "Dissemble the $RAID0_DEV Raid Device"
      # Stop the array
      mdadm --stop $RAID0_DEV > /dev/null 2>&1 || true

      # If the raid device itself is already mounted, get off it.
      umount -l $RAID0_DEV > /dev/null 2>&1 || true

      mdadm --remove $RAID0_DEV > /dev/null 2>&1 || true

      #zero the superblock FOR EACH drive
      mdadm --zero-superblock $EPHEMERAL0_DEV > /dev/null 2>&1 || true
      mdadm --zero-superblock $EPHEMERAL1_DEV > /dev/null 2>&1 || true

      # If the first second device itself is already mounted, get off it.
      # After breaking down the raid device
      umount -l $EPHEMERAL1_DEV > /dev/null 2>&1 || true
    fi

    # If the first ephemeral device itself is already mounted, get off it.
    # After potentially breaking down a raid device
    umount -l $EPHEMERAL0_DEV > /dev/null 2>&1 || true
    # in case the mount is setup in /etc/fstab
    sed -i '/\/dev\/xvdb/d' /etc/fstab > /dev/null 2>&1 || true

    log "Everything should be unmounted now"
}

####
## Partitions the device for us
####
function partitionBaseDevice {
    log "Partitioning $BASEDEVICE"
    # Setup swap space and ephemereal space
     > /dev/null 2>&1 || true
    sfdisk -f $BASEDEVICE < /var/lib/ReproConnect/Utilities/etc/init/ephemeralParitions
}

####
## Sets up swap space for us
####
function setupSwap {
    log "Setting up swap space"
    # Format swap space
    mkswap $SWAP_PART
    # And mount it
    swapon $SWAP_PART
}

####
## Formats the data partition for us
####
function formatData {
    log "Formating $DATA_PART"
    
    # Format Ephemeral Space
    mkfs.ext4 -b 4096 -E stride=64,stripe-width=128 $DATA_PART
}

####
## Sets up the data partition, creating directories, mounting /tmp
## and all sorts of fun stuff
####
function setupData {
    log "Setting Up Data"
    # If the data partition is not mounted on the data dir
    if [ `mount|grep -c "$DATA_PART on $EPHEMERAL_DATA_DIR"` -eq 0 ]; then
        log "The data partition $DATA_PART is not mounted on $EPHEMERAL_DATA_DIR"
        # Check if we have a mountpoint
        if [ ! -d $EPHEMERAL_DATA_DIR ] ; then
            log "Creating $EPHEMERAL_DATA_DIR"
            mkdir -p $EPHEMERAL_DATA_DIR
        fi
        log "Mounting $DATA_PART on $EPHEMERAL_DATA_DIR"
        mount "$DATA_PART" "$EPHEMERAL_DATA_DIR" > /dev/null 2>&1 || true
        chmod 777 "$EPHEMERAL_DATA_DIR"
        chown $data_user:$data_group "$EPHEMERAL_DATA_DIR"
    fi
    log "The data partition $DATA_PART is mounted on $EPHEMERAL_DATA_DIR"
    # If our log directory doesn't exist, create it
    if [ ! -d "$EPHEMERAL_LOG_DIR" ] ; then
        log "Setting up $EPHEMERAL_LOG_DIR"
        mkdir -p "$EPHEMERAL_LOG_DIR"
        chmod -R 775 "$EPHEMERAL_LOG_BASE_DIR"
        chown -R $data_user:$data_group "$EPHEMERAL_LOG_BASE_DIR"
    fi

    # If our tmp directory doesn't exist, create it
    if [ ! -d "$EPHEMERAL_TMP_DIR" ] ; then
        log "Setting up $EPHEMERAL_TMP_DIR"
        mkdir -p "$EPHEMERAL_TMP_DIR"
        chmod -R 1777 "$EPHEMERAL_TMP_DIR"
        chown -R $data_user:$data_group "$EPHEMERAL_TMP_DIR"
        log "Copying /tmp/* to $EPHEMERAL_TMP_DIR"
        cp -rp /tmp/* $EPHEMERAL_TMP_DIR
        log "Mounting $EPHEMERAL_TMP_DIR on /tmp"
        mount --bind $EPHEMERAL_TMP_DIR /tmp  > /dev/null 2>&1 || true
    fi
}


####
## Creates a raid0 device for us
####
function createRaid {
    log "Creating the Raid Device"
    # Create the Raid Device
    yes | mdadm --create $RAID0_DEV --level=0 -c256 --raid-devices=2 $EPHEMERAL0_DEV $EPHEMERAL1_DEV
    echo "DEVICE $EPHEMERAL0_DEV $EPHEMERAL1_DEV" > /etc/mdadm/mdadm.conf
    mdadm --detail --scan >> /etc/mdadm/mdadm.conf
    # update initramfs so it contains your mdadm.conf settings during boot
    sudo update-initramfs -u
}
## End of functions
##########################################################################

# Call main, and exit
main
exit 0