influx_restore.bash

gistfile1.txt

#!/bin/bash
#set -e
# Script to restore InfluxDB data from first full backup found before $RESTORE_DATE until $RESTORE_DATE.

# Pre-requisites:
# RESTORE_DATE             : Date is set in format YYYYMMDDTHHMMSSZ (UTC time). Can be "latest".
# AZ_ENV                   : Environment to work in.
# DATABASES                : Databases to import.
# TO_DATABASES         : Databases to import to new databases.
# TENANT_ID                : Tenant ID to work with.
# AZCOPY_SPA_CLIENT_SECRET : Sercret for SP
# AZ_USERNAME              : Service Principal for Azure login
# AZ_RG                    : Azure Resource group from where to retrieve Backups from.
# AZ_STORAGEACCOUNT        : Azure Storage account where to retrieve Backups from.
# INFLUXDB_HOST            : Host InfluxDB
# INFLUXDB_PORT            : Port InfluxDB
# INFLUX_BR_PORT           : Port for Backup Restore InfluxDB

SLACK_URI=""
LOCAL_DIR="/tmp"

# Put DATABASES and TO_NEW_DATABASES into array
ARRAY_DB=($DATABASES)
echo " List of Databases to restore: ${ARRAY_DB[*]}"
ARRAY_TO_DB=($TO_DATABASES)
echo " List of Databases to restore IN: ${ARRAY_TO_DB[*]}"


## Be sure to use kubeconfig that match the targeted environment to restore in.
#kubectl port-forward service/influxdb1 8088:8088 -n data > /dev/null &
#kubectl port-forward service/influxdb1 8086:8086 -n data > /dev/null &

if [[ $RESTORE_DATE == "latest" ]]
then
    RESTORE_DATE=$(date -u +%Y%m%dT%H%M%SZ)
fi

az login --tenant $TENANT_ID --service-principal -u $AZ_USERNAME -p $AZCOPY_SPA_CLIENT_SECRET > /dev/null
if [[ $? -ne 0 ]]
then
    curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: issue during login to Azure storage account for retrieving INFLUXDB databases ['"$DATABASES"'] : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI
    exit 1
fi

KEY=$(az storage account keys list --account-name $AZ_STORAGEACCOUNT --resource-group $AZ_RG --query '[0].value' --output tsv)
if [[ $KEY == "" ]]
then
    curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: issue on Azure storage account Key retrieval during recovery process of INFLUXDB databases ['"$DATABASES"'] : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI
    exit 1
fi


echo " Start of InfluxDB restore for a recovery at ${RESTORE_DATE} from ${ARRAY_DB[*]} respectively to ${ARRAY_TO_DB[*]}... - `date +%Y/%m/%d-%H:%M:%S`"

for directory in `az storage fs file list --account-name $AZ_STORAGEACCOUNT --account-key $KEY -f influxdb --recursive=false 2>/dev/null | jq -r '.[].name' | sort -r`
do
    if [[ "$(echo $directory | cut -d'-' -f1)" < "$RESTORE_DATE" ]]
    then
        if [[ $directory != *"-full" ]]
        then
            LIST="$LIST $directory"
        elif [[ $directory == *"-full" ]]
        then
            LIST="$LIST $directory"
            break
        fi
    fi
done

LIST=$(echo "$LIST" | awk '{ for (i=NF; i>1; i--) printf("%s ",$i); print $1; }')
echo " List of Directories to retrieve for restoring data at ${RESTORE_DATE}: $LIST"

export AZCOPY_SPA_CLIENT_SECRET
azcopy login --service-principal --application-id $AZ_USERNAME --tenant-id $TENANT_ID

for selected_directory in $LIST
do
    echo " Copy $selected_directory locally ..."
    azcopy cp "https://${AZ_STORAGEACCOUNT}.blob.core.windows.net/influxdb/$selected_directory" "${LOCAL_DIR}/" --recursive=true
    if [[ $? -ne 0 ]]
    then
        curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: issue during copy of INFLUXDB Backup https://'"${AZ_STORAGEACCOUNT}"'.blob.core.windows.net/influxdb/'"$selected_directory"' locally : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI
        echo " Local copy of $selected_directory failed."
        exit 1
    else
        echo " Local copy of $selected_directory completed."
    fi

    TS=$(echo $selected_directory | cut -d'-' -f1)

    #ORI: for database in $DATABASES
    for index in ${!ARRAY_DB[*]}
    do
        #influx -host $INFLUXDB_HOST -port $INFLUXDB_PORT -execute "SHOW DATABASES" | grep ^${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}$ > /dev/null
        influx -execute "SHOW DATABASES" | grep ^${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}$ > /dev/null
        DB_EXISTS_CODE=$?
        if [[ $DB_EXISTS_CODE -eq 0 ]]
        then
            #Direct load for temporary side load DB
            ERROR_SIDE_LOAD=1
            ERR_COUNT=0
            while [[ $ERROR_SIDE_LOAD -ne 0 && $ERR_COUNT -lt 3 ]]
            do
                echo "This is try number $((ERR_COUNT+1)) for direct load for temporary side load DB."
		        sleep 3
                #influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} ${LOCAL_DIR}/$selected_directory
                influxd restore -portable -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} ${LOCAL_DIR}/$selected_directory
                ERROR_SIDE_LOAD=$?
                ERR_COUNT=$((ERR_COUNT+1))
            done

            if [[ $ERROR_SIDE_LOAD -ne 0 ]]
            then
                curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: influxd restore command failed for '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"'-restored-'"${TS}"' : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI
                echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} failed."
                exit 1
            else
                echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} completed."
            fi

	        #Side load for existing DB
	        ERROR_SIDE_LOAD=1
	        ERR_COUNT=0
	        while [[ $ERROR_SIDE_LOAD -ne 0 && $ERR_COUNT -lt 3 ]]
            do
		    echo "This is try number $((ERR_COUNT+1)) for side load."
		    sleep 3
            #influx -host $INFLUXDB_HOST -port $INFLUXDB_PORT -database ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} -execute "SELECT * INTO ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}..:MEASUREMENT FROM /.*/ GROUP BY *"
            influx -database ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} -execute "SELECT * INTO ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}..:MEASUREMENT FROM /.*/ GROUP BY *"
		    ERROR_SIDE_LOAD=$?
		    ERR_COUNT=$((ERR_COUNT+1))
            done

            if [[ $ERROR_SIDE_LOAD -ne 0 ]]
            then
                curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: injection of '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"'-restored-'"${TS}"' into '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"' failed : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI
                echo " Injection of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} into ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]} failed."
                exit 1
            else
                echo " Injection of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} into ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]} completed."
            fi
            #influx -host $INFLUXDB_HOST -port $INFLUXDB_PORT -execute "DROP DATABASE \"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS}\"" > /dev/null
            influx -execute "DROP DATABASE \"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS}\"" > /dev/null
        else
	    #Direct load for NON existing DB
            ERROR_SIDE_LOAD=1
            ERR_COUNT=0
            while [[ $ERROR_SIDE_LOAD -ne 0 && $ERR_COUNT -lt 3 ]]
            do
                echo "This is try number $((ERR_COUNT+1)) for direct load in NON existing target Database."
	        	sleep 3

                if [[ -z ${ARRAY_TO_DB[$index]} ]];
                then
                    #influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db ${ARRAY_DB[$index]} ${LOCAL_DIR}/$selected_directory
                    influxd restore -portable -db ${ARRAY_DB[$index]} ${LOCAL_DIR}/$selected_directory
                else
                    #influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]} ${LOCAL_DIR}/$selected_directory
                    influxd restore -portable -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]} ${LOCAL_DIR}/$selected_directory
                fi

                #ORI: influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db $database ${LOCAL_DIR}/$selected_directory
                ERROR_SIDE_LOAD=$?
                ERR_COUNT=$((ERR_COUNT+1))
            done

            if [[ $ERROR_SIDE_LOAD -ne 0 ]]
            then
                curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: influxd restore command failed for '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"' : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI
                echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} failed."
                exit 1
            else
                echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} completed."
            fi
        fi
    done

    rm -rf ${LOCAL_DIR}/$selected_directory
done

#kill $(jobs -p)

curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopInfo", "text":"['"$AZ_ENV"'] OK: Restore for recovery at '"$RESTORE_DATE"' from InfluxDB databases ['"${ARRAY_DB[*]}"'] into InfluxDB databases ['"${ARRAY_TO_DB[*]}"'] completed : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":white_check_mark:"}' $SLACK_URI
echo "OK: Restore Completed."

exit 0