A universal script that runs in the background on both the primary and secondary servers and automatically switches staked identity if delinquent status is detected, even if the connection is lost.

solana-validator-auto-swap-identity.sh

#!/bin/bash

# A universal script that runs in the background on both the primary and secondary servers and automatically switches staked identity if delinquent status is detected, even if the connection is lost.
# Usage: Run this script in the background on both validators.
# Solana validator should always start with UNSTAKED identity on both servers. The script will switch to STAKED identity automatically.
#
# BUGS STILL CAN OCCUR. USE IT AT YOUR OWN RISK.
#
# Author: STEALTH
# Validator ID: 93Q99nhdKjuSe6WNXgMBbC3s8QVQEAoHKt91PNRkUkMn
#
# Last update: 04.06.2025

# Settings
SOLANA_PATH="$HOME/.local/share/solana/install/active_release/bin"
STAKED_IDENTITY_KEYPAIR=$(cat "/path/to/staked_identity.json")
STAKED_IDENTITY_PUBKEY="$($SOLANA_PATH/solana-keygen pubkey /dev/stdin <<< $STAKED_IDENTITY_KEYPAIR)"
UNSTAKED_IDENTITY_KEYPAIR=$(cat "/path/to/unstaked_identity.json")
PREV_ACTIVE_IDENTITY_PUBKEY=""
RPC_URL="$($SOLANA_PATH/solana config get | grep RPC | awk '{print $3}')"
CHECK_INTERVAL=10
SWITCH_DELAY_PRIMARY=300
SWITCH_DELAY_SECONDARY=30
# Telegram bot alerts
BOT_TOKEN=""
CHAT_ID=""


telegram_message() {
    curl --header 'Content-Type: application/json' --request 'POST' --data '{"chat_id":"'"$CHAT_ID"'","text":"'"$1"'"}' "https://api.telegram.org/bot$BOT_TOKEN/sendMessage"
}

log_message() {
    echo "[$(date -u +"%F %T")] $1" >> $HOME/solana/auto-swap-identity.log
}

log_message "Starting Solana Automated Identity Swap script..."

if [[ ! -f "$SOLANA_PATH/solana" || ! -f "$SOLANA_PATH/agave-validator" ]]; then
    log_message "Error: Solana binaries not found in $SOLANA_PATH"
    exit 1
fi

if [[ $STAKED_IDENTITY_KEYPAIR == "" ]]; then
    log_message "Error: STAKED identity not found."
    exit 1
fi

if [[ $UNSTAKED_IDENTITY_KEYPAIR == "" ]]; then
    log_message "Error: UNSTAKED identity not found."
    log_message 'Use "solana-keygen new --no-bip39-passphrase -s -o unstaked_identity.json" to generate new unstaked identity.'
    exit 1
fi

check_internet_connection() {
    local servers=("google.com" "cloudflare.com" "github.com")
    local attempts=3
    local interval=10

    for ((i=0; i<attempts; i++)); do
        for server in "${servers[@]}"; do
            if ping -c 1 $server &> /dev/null; then
                return 0
            fi
        done

        if curl -s --head http://www.google.com | grep "200 OK" > /dev/null; then
            return 0
        fi

        log_message "Attempt #$((i+1))/$attempts: Internet connection unavailable. Retry in $interval seconds..."
        sleep $interval
    done

    return 1
}

check_local_rpc() {
    if $SOLANA_PATH/agave-validator --ledger $LEDGER_PATH contact-info > /dev/null; then
        RPC_PORT="$(systemctl status solana | grep -o -- "--rpc-port [^ ]*" | awk '{print $2}')"
        RPC_URL="http://localhost:$RPC_PORT"
        log_message "Local RPC: OK."
        return 0
    else
        # RPC_URL="$($SOLANA_PATH/solana config get | grep RPC | awk '{print $3}')"
        log_message "Local RPC: ERROR. Validator startup is not complete."
        return 1
    fi
}

check_delinquency() {
    local attempts
    local interval
    
    if [[ "$PREV_ACTIVE_IDENTITY_PUBKEY" == "$STAKED_IDENTITY_PUBKEY" && "$PREV_ACTIVE_IDENTITY_PUBKEY" != "$ACTIVE_IDENTITY_PUBKEY" ]]; then
        log_message "Validator switched from staked to unstaked identity."
        log_message "Waiting $SWITCH_DELAY_PRIMARY seconds before checking delinquency status..."
        sleep $SWITCH_DELAY_PRIMARY

        attempts=$(echo "$SWITCH_DELAY_PRIMARY/$CHECK_INTERVAL" | bc)
        interval=$(echo "$SWITCH_DELAY_PRIMARY/$attempts" | bc)
    else
        attempts=$(echo "$SWITCH_DELAY_SECONDARY/$CHECK_INTERVAL" | bc)
        interval=$(echo "$SWITCH_DELAY_SECONDARY/$attempts" | bc)
    fi

    for ((i=0; i<attempts; i++)); do
        # Check local RPC before each delinquency check
        if ! check_local_rpc; then
            log_message "Skipping delinquency check due to local RPC error."
            return 1
        fi 

        local delinquent_status="$($SOLANA_PATH/solana --url $RPC_URL validators --output json-compact | jq --arg IDENTITY "$STAKED_IDENTITY_PUBKEY" '.validators[] | select(.identityPubkey==$IDENTITY) ' | jq .delinquent)"

        if [[ $delinquent_status == "false" ]]; then
            return 1
        fi

        log_message "Attempt #$((i+1))/$attempts: $STAKED_IDENTITY_PUBKEY is delinquent! Retry in $interval seconds..."
        sleep $interval
    done

    return 0
}

switch_identity() {
    local identity=$1
    local action=$2
    
    if [[ $action == "add" ]]; then
        log_message "Removing $TOWER_PATH"
        rm -f $TOWER_PATH
        log_message "$($SOLANA_PATH/agave-validator --ledger $LEDGER_PATH authorized-voter $action <<< $identity)"
    else
        log_message "$($SOLANA_PATH/agave-validator --ledger $LEDGER_PATH authorized-voter $action)"
    fi

    log_message "$($SOLANA_PATH/agave-validator --ledger $LEDGER_PATH set-identity <<< $identity)"
    log_message "Exiting..."
#    log_message "Waiting $SWITCH_DELAY_PRIMARY seconds before continuing..."
#    sleep $SWITCH_DELAY_PRIMARY
    telegram_message "\u26A0 $(hostname -I) has switched identity to:\n$($SOLANA_PATH/solana-keygen pubkey /dev/stdin <<< $identity)\n\nExiting the script..."
    exit 1
}

# Main loop
while true; do
    LEDGER_PATH="$(systemctl status solana | grep -o -- "--ledger [^ ]*" | awk '{print $2}')"
    TOWER_PATH="$(systemctl status solana | grep -o -- "--tower [^ ]*" | awk '{print $2}')"

#    if [ -z "$LEDGER_PATH" ]; then
#        log_message "Error: Ledger path not found."
#        exit 1
#    fi

    if [ -z "$TOWER_PATH" ]; then
        TOWER_PATH="$LEDGER_PATH/tower-1_9-$STAKED_IDENTITY_PUBKEY.bin"
    else
        TOWER_PATH="$TOWER_PATH/tower-1_9-$STAKED_IDENTITY_PUBKEY.bin"
    fi

    if $SOLANA_PATH/agave-validator --ledger $LEDGER_PATH contact-info > /dev/null; then
        RPC_PORT="$(systemctl status solana | grep -o -- "--rpc-port [^ ]*" | awk '{print $2}')"
        RPC_URL="http://localhost:$RPC_PORT"
        log_message "Local RPC: OK."
    else
        log_message "Local RPC: ERROR. Validator startup is not complete. Retry in $(echo "$CHECK_INTERVAL*3" | bc) seconds..."
        echo ""
        sleep $(echo "$CHECK_INTERVAL*3" | bc)
        continue
    fi

    ACTIVE_IDENTITY_PUBKEY="$($SOLANA_PATH/agave-validator --ledger $LEDGER_PATH contact-info | grep Identity | awk '{print $2}')"
    
    log_message "Staked Identity: $STAKED_IDENTITY_PUBKEY"
    log_message "Active Identity: $ACTIVE_IDENTITY_PUBKEY"
    log_message "Prev   Identity: $PREV_ACTIVE_IDENTITY_PUBKEY"

    if check_internet_connection; then 
        log_message "Internet connection: OK."
        if check_delinquency; then
            if [[ $ACTIVE_IDENTITY_PUBKEY == $STAKED_IDENTITY_PUBKEY ]]; then
                switch_identity $UNSTAKED_IDENTITY_KEYPAIR "remove-all"
            else
                switch_identity $STAKED_IDENTITY_KEYPAIR "add"
            fi
        else
            log_message "$STAKED_IDENTITY_PUBKEY is not delinquent."
        fi
    else 
        log_message "Internet connection: ERROR."
        if [[ $ACTIVE_IDENTITY_PUBKEY == $STAKED_IDENTITY_PUBKEY ]]; then 
            switch_identity $UNSTAKED_IDENTITY_KEYPAIR "remove-all"
        else
            log_message "Unstaked identity is active, do nothing."
        fi
    fi

    # Update previous active identity
    PREV_ACTIVE_IDENTITY_PUBKEY=$ACTIVE_IDENTITY_PUBKEY

    log_message "Waiting $CHECK_INTERVAL seconds before next run..."
    echo ""
    sleep $CHECK_INTERVAL
done

How to use:

Create a screen session in the background and run the script within it.

screen -dmS auto-swap-identity ./solana-validator-auto-swap-identity.sh

Terminate the screen session.

screen -S auto-swap-identity -X quit

Read the log file.

tile -f $HOME/solana/auto-swap-identity.log

NOTE: After switching identities, the script sends an alert message to the Telegram bot and terminates. The script still needs debugging so that it can run in the background without restarting.

* After running the script, you may want to delete staked_identity.json from the server for security reasons.