Hadoop root drive swap

gistfile1.txt

#!/bin/bash
set -x # verbosity

# This script is meant to be run from inside alchemy linux, our PXE rescue system.
# To boot into alchemy, connect to a serial console with ipmitool sol activate, and set the bootdev with 'chassis bootdev pxe'
# The default user:pass is alchemy:transmute
# This script covers swapping out the OS drive of a hadoop node, and is very coupled to this task. The individual sections however may be reused.
# This script should never be run without being watched, and should never be run all at once. Run each section on its own. Ye be warned.

# The first step is to pick a data drive to remove. We'll need to find out the serial for that drive. The only mapping between the physical and logical here is done through "megaraid target id's", which map to the scsi sub slot. Sound confusing? Don't worry, that's because it is, so we'll confuse it more by using the folliwng magical regex:

MEGARAID_CMD=/opt/megacli/megacli

SWAPOUT=sdf # the data drive we want to remove
data_targetid=$(ls -l /dev/disk/by-path/ | grep $SWAPOUT | head -n1 | sed 's/.*scsi-0:[0-9]:\([0-9]*\):.*/\1/g')
serial=$($MEGARAID_CMD -LdPdInfo -aALL  | grep "Target Id: $data_targetid" -A53 | tail -n1) # the serial number is likely the last field.
echo $serial

# Here we'll detect the newly inserted drive, and create it as a single drive raid0. Yes, megaraid is stupid.
slot=`sudo $MEGARAID_CMD -PDList -aAll | grep Unconfigured -B 17 | head -n1 | cut -d ' ' -f 3`
sudo $MEGARAID_CMD -CfgLdAdd -r0 [252:$slot] -a0

# Now we'll create the partition table, and create a full disk partition. Blank lines are intentional, as they pass the default response
NEWDEVICE=`lsblk | grep 893 | awk '{print $1}'` #893 is a magic number, the size of a 1TB crucial M500 hard drive.
echo $NEWDEVICE
fdisk /dev/$NEWDEVICE <<EOF
o
n




w
EOF

# Now we'll create the LVM physical, volume, and logical groups
pvcreate /dev/${NEWDEVICE}1
vgcreate vgssd /dev/${NEWDEVICE}1

lvcreate -L 40G -nroot vgssd
lvcreate -L 100G -nswap vgssd
lvcreate -L 50G -nu vgssd
lvcreate -L 700G -ntmp vgssd

# And format them
mkfs.ext4 /dev/mapper/vgssd-root
mkfs.ext4 /dev/mapper/vgssd-u
mkfs.ext4 -i 4096 /dev/mapper/vgssd-tmp  # the -i 4096 here says to use inodes that are 1/4 the default size, so we get more of them for tmp
mkswap /dev/mapper/vgssd-swap

# Now we'll mount the drives we made
mkdir /mnt/new /mnt/u /mnt/tmp /mnt/old /mnt/boot
mount /dev/mapper/vgssd-tmp /mnt/tmp
mount /dev/mapper/vgssd-u /mnt/u
mount /dev/mapper/vgssd-root /mnt/new
mount /dev/mapper/lvroot-root /mnt/old
mount /dev/sda1 /mnt/boot
chmod -R 1777 /mnt/tmp # set everyone and sticky bit

# And copy some data we care about, and remove what we don't
mv /mnt/old/u/* /mnt/u
rm -rf /mnt/old/tmp/* # cleared on startup anyways, so let's not copy it
cp -r /mnt/boot/* /mnt/old/boot # keep the kernel or else we're fucked
#tarpipes ftw
cd /mnt/old

# NOTE: make sure that the old root isn't bigger than the new root First!!!
# Here's the magic source - a tarpipe to copy everything, preserving the permissions of all files. There are other ways, but I like this best.
tar -cpf - . | sudo tar -C /mnt/new -xpf -


# Manually fix /etc/fstab to remove crappy entries, then append this:
#/dev/mapper/vgssd-root /               ext4    errors=remount-ro,discard 0       1
#/dev/mapper/vgssd-swap none            swap    sw              0       0
#/dev/mapper/vgssd-tmp  /tmp ext4 nobarrier,discard,noatime,nodiratime 1 2
#/dev/mapper/vgssd-u    /u ext4 discard 1 2

# Now we chroot into the new root system
mount --rbind /dev /mnt/new/dev
mount --rbind /sys /mnt/new/sys
mount -t proc none /mnt/new/proc

# And make sure we haven't confused the fuck out of grub
mkdir  /mnt/new/tmp
chmod -R 1777 /mnt/new/tmp # set everyone and sticky bit
mkdir  /mnt/new/run/lock
chroot /mnt/new grub-install /dev/${NEWDEVICE}
chroot /mnt/new update-grub2

# Clean up
umount -l /mnt/new/dev
umount -l /mnt/new/sys
umount -l /mnt/new/proc

# Set the boot device to the new drive in the raid controller
targetid=$(ls -l /dev/disk/by-path/ | grep $NEWDEVICE | head -n1 | sed 's/.*scsi-0:[0-9]:\([0-9]*\):.*/\1/g')
$MEGARAID_CMD -AdpBootDrive -set -L${targetid} -a0


# Now we verify that the system boots properly by trying to boot into it - we don't want to remove the old root drive until we know the new one works.
# Once that is done, we'll determine the serial of the old root drive, just like we did earlier

SWAPOUT=sda # the data drive we want to remove
old_root_targetid=$(ls -l /dev/disk/by-path/ | grep $SWAPOUT | head -n1 | sed 's/.*scsi-0:[0-9]:\([0-9]*\):.*/\1/g')
serial=$($MEGARAID_CMD -LdPdInfo -aALL  | grep "Target Id: $old_root_targetid" -A53 | tail -n1) # the serial number is likely the last field.
echo $serial

## Once the swap has been completed, the last step will be to tell the raid controller that the drive has moved, and needs to be reconfigured.
# The drive is ACTUALLY Configured, but the raid controller is to stupid te realize that it moved

$MEGARAID_CMD -CfgForeign -Clear -a0
unconf_slot=$($MEGARAID_CMD -PDList -aAll | grep  nconf -B17 | head -n1 | awk '{print $3}')
$MEGARAID_CMD -PDMakeGood -PhysDrv [252:$unconf_slot] -a0 # it's usually ok if this fails, it should fail if the drive is good.
$MEGARAID_CMD -CfgLdadd -r0 [252:$unconf_slot] -a0

# Now we just need to enable the drives again!
sed -i '/^#UUID.* /s/^#//' /etc/fstab
mount -a

# And we're done, just verify all the drives arou mounted, and we should be ready to keep hadooping.