Bash script for checking disk health for a Storj Storagenode. The disk is automatically remounted if the error "Transport endpoint is not connected" is encountered. Then the Storj Storagenode docker service is checked and started after the disk is mounted.

storj_disk_check.sh

#!/bin/bash

# Add to crontab for periodic check:
# Storj disk check every 4 hours - remounts and restarts storagenode if disk is unavailable
# 0 */4 * * * /bin/bash /root/scripts/storj_disk_check.sh

# Variables
SCRIPT_NAME=`basename "$0"`
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
LOG_FILE="$SCRIPT_DIR/$(echo $SCRIPT_NAME | cut -d'.' -f1).log"
LOG_LEVEL=2 # 1 - ERROR, 2 - INFO, 3 - DEBUG
MOUNT_POINT="/storj"
MOUNT_ACTIVE=1

# Logging methods
info() {
    if [[ "$1" && $LOG_LEVEL -ge 2 ]]; then
        echo info
        echo -e "<$(date)> <INFO> <$1>" >> $LOG_FILE
    fi
}

error() {
    if [[ "$1" && $LOG_LEVEL -ge 1 ]]; then
        echo error
        echo -e "<$(date)> <ERROR> <$1>" >> $LOG_FILE
    fi
}

debug() {
    if [[ "$1" && $LOG_LEVEL -ge 3 ]]; then
        echo debug
        echo -e "<$(date)> <DEBUG> <$1>" >> $LOG_FILE
    fi
}

# Check if log file exists, touch if not
if [ ! -f "$LOG_FILE" ]; then
        touch $LOG_FILE
fi

# Debug logging
debug "SCRIPT_NAME=$SCRIPT_NAME"
debug "SCRIPT_DIR=$SCRIPT_DIR"
debug "LOG_FILE=$LOG_FILE"
debug "LOG_LEVEL=$LOG_LEVEL"
debug "MOUNT_POINT=$MOUNT_POINT"

# Try to cd to mount point
cd $MOUNT_POINT && MOUNT_ACTIVE=0

# If mount is not active, remount and restart storagenode
if [ $MOUNT_ACTIVE -ne 0 ]; then
        error "Mount point $MOUNT_POINT is not active. Doing a lazy unmount..."
        fusermount -uz $MOUNT_POINT
        if [ $? -ne 0 ]; then
                error "Could not unmount. Exiting..."
                exit 1
        else
                info "Unmount successful. Remounting..."
                mount $MOUNT_POINT
                if [ $? -ne 0 ]; then
                        error "Could not remount $MOUNT_POINT. Exiting..."
                        exit 2
                else
                        info "Remount successful. Starting Storj Storagenode..."
                        docker start storagenode
                        if [ $? -ne 0 ]; then
                                error "Could not start Storj Storagenode. Exiting..."
                                exit 3
                        else
                                info "Storj Storagenode started successfully. Exiting..."
                        fi
                fi
        fi
else
        info "Mount point $MOUNT_POINT is active. Checking if Storj Storagenode is running..."
        docker ps | grep storagenode
        if [ $? -ne 0 ]; then
                error "Storj Storagenode is not running. Starting Storj Storagenode..."
                docker start storagenode
                if [ $? -ne 0 ]; then
                        error "Could not start Storj Storagenode. Exiting..."
                        exit 3
                else
                        info "Storj Storagenode started successfully. Exiting..."
                fi
        else
                info "Storj Storagenode is running. Exiting..."
        fi
fi

exit 0