Skip to content

Instantly share code, notes, and snippets.

@erincerys
Last active December 26, 2015 14:29
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save erincerys/7165709 to your computer and use it in GitHub Desktop.
Save erincerys/7165709 to your computer and use it in GitHub Desktop.
Shutdown a server if a fusefs connection become unreliable
#!/bin/bash
# If a fusefs connection has been lost, shutdown the server and optionally log the event to an S3 object.
# I use this AWS EC2 instances in auto-scaling groups. If it shuts down, it gets terminated by the ASG, and replaced with a fresh one.
# Might be some fringe cases in the error handling code that haven't been encountered or thought of. Hence, might not be 100% reliable, so use at your own risk
cd /root
# Create lock
lockfile='/root/fscheck.lock'
lockpresent=$(cat $lockfile 2>/dev/null)
if [ "$lockpresent" ] ; then exit 1 ; fi
echo 1 > $lockfile
fusetype='sshfs'
log2s3=1
guid=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
testfile="/path/to/mountpoint/of/fusefs/testing-${guid}"
instlog='asg-instances.log'
instid=`wget -q -t 3 -T 2 -O - http://169.254.169.254/latest/meta-data/instance-id || echo 'fail'`
azplacement=`wget -q -t 3 -T 2 -O - http://169.254.169.254/latest/meta-data/placement/availability-zone || echo 'fail'`
po () {
if [ $log2s3 -eq 1 ] ; then
s3cmd get --config /root/.s3cfg --force s3://live-mls-meta/${instlog} ./
if [ "$instid" != 'fail' ] ; then
echo "Instance $instid in $azplacement entered an unhealthy state ($err)" >> ${instlog}
else
echo "Instance (unknown ID) entered an unhealthy state ($err)" >> ${instlog}
fi
s3cmd put --config /root/.s3cfg ./${instlog} s3://live-mls-meta/
fi
rm $lockfile
sudo shutdown -h now
exit 1
}
# Mount test
if [ $(mount | grep -c $fusetype) -eq 0 ] ; then
err='mount check'
po
fi
# File creation test
touch $testfile
creationtest=$(ls -1 $testfile 2>&1)
if [[ "$creationtest" ]] && [[ $(echo "$creationtest" | egrep -c "(No such|Permission denied)") -eq 1 ]] ; then
err='file creation'
po
fi
# Write test
if [ $(sudo -u www-data -H bash -c "echo 'hello world' > $testfile" 2>&1) ] ; then
err='file write'
po
fi
# Read test
length=$(cat $testfile | wc -c)
if [ "$length" -eq 0 ] || [ "$length" -ne "$length" ] ; then
err='file read'
po
fi
# Delete test
if [ $(rm -f $testfile 2>&1) ] ; then
err='file delete'
po
fi
rm $lockfile
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment