solana-restart-failure-investigation-2022.11.06/same-shred-different-bank-again

check-snapshots.py

#!/usr/bin/python3

import json
import os.path
import urllib.error
import urllib.request

_LAMPORTS_PER_SOL = 1000000000
_HTTP_TIMEOUT = 5  # seconds

_RIGHT_SHRED_VERSION = 4711
_RIGHT_SNAPSHOT = 'snapshot-160991176-5GMY4MtGzDnXYq9AVssRBxVyRj6YVRMjxfjZtqVBeP97.tar.zst'

# wget -O destaked.txt 'https://gist.githubusercontent.com/CrazySerGo/9f14a3d4d5d2efca132dc9433b16a03e/raw/5e31452a40c1215f91d1ad962047e2f69c0b5731/Exclusion%2520list%252004.11.2022'
_DESTAKED_DUMP_PATH = 'destaked.txt'
# solana -u ${YOUR_RPC} --output json validators >validators.json
_VALIDATORS_DUMP_PATH = 'validators.json'
# curl ${YOUR_RPC} -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getClusterNodes"}' >cluster-nodes.json
_CLUSTER_NODES_DUMP_PATH = 'cluster-nodes.json'


def _main():
    destaked_vote_accs = _read_destaked(_DESTAKED_DUMP_PATH)
    total_active_stake, validator_vote_accs = _read_validators(_VALIDATORS_DUMP_PATH)
    nodes = _read_cluster_nodes(_CLUSTER_NODES_DUMP_PATH)

    url_opener = _create_opener()

    stake_with_working_rpc = 0
    stake_with_working_rpc_and_bad_snapshot = 0
    n_working_rpc_validators = 0
    bad_validators = []
    n = 0
    for node in nodes:
        n += 1
        if node['shredVersion'] != _RIGHT_SHRED_VERSION:
            # Different shred version.
            continue
        if node.get('rpc') is None:
            # RPC is not enabled. We are not able to check the snapshot in this case.
            continue
        validator_id = node['pubkey']
        if validator_id not in validator_vote_accs:
            # Unknown validator.
            continue
        validator_vote = validator_vote_accs[validator_id]
        validator_stake = validator_vote['stake']
        if validator_stake == 0:
            # No stake.
            continue

        # Sanity check:
        assert validator_vote['pubkey'] not in destaked_vote_accs, \
            f'Staked validator is in destaked list: {validator_id}'

        node_rpc = node['rpc']
        node_rpc = f'http://{node_rpc}'
        print(f'{n}/{len(nodes)} {validator_id} {node_rpc}')

        snap_name = _get_current_snapshot_name(url_opener, node_rpc)
        if snap_name is None:
            # Unable to get snapshot name (RPC advertised but not functioning).
            continue
        stake_with_working_rpc += validator_stake
        n_working_rpc_validators += 1
        if snap_name == _RIGHT_SNAPSHOT:
            # Good validator!
            continue

        stake_with_working_rpc_and_bad_snapshot += validator_stake
        bad_validators.append(
            dict(
                identity=validator_id,
                vote=validator_vote['pubkey'],
                stake=validator_stake,
                snap=snap_name,
                slot=_get_current_slot(url_opener, node_rpc)))

    print(f'Total stake with advertised and working RPC: {stake_with_working_rpc / _LAMPORTS_PER_SOL:.02f} SOL')
    print(f'Total stake with advertised and working RPC but bad snapshot:'
          f' {stake_with_working_rpc_and_bad_snapshot / _LAMPORTS_PER_SOL:.02f} SOL'
          f' / {stake_with_working_rpc_and_bad_snapshot / stake_with_working_rpc * 100.0:.02f}%')
    print(f'{len(bad_validators)} validators with wrong snapshot out of {n_working_rpc_validators} '
          f'validators with working RPC:')
    for validator in bad_validators:
        print(f"{validator['identity']} (vote {validator['vote']}, stake"
              f" {validator['stake'] / _LAMPORTS_PER_SOL:.02f} SOL"
              f" / {validator['stake'] / stake_with_working_rpc * 100.0:.02f}%): {validator['snap']},"
              f" slot {validator['slot']}")


def _get_current_snapshot_name(opener, rpc_url):
    url = rpc_url + '/snapshot.tar.bz2'
    request = urllib.request.Request(
        url=url,
        method='HEAD',
    )

    try:
        response = opener.open(request, timeout=_HTTP_TIMEOUT)
    except urllib.error.URLError:
        return None

    assert response.status == 303, \
        f'Wrong status when requesting HEAD on "{url}": {response.status} (should be 303)'
    assert 'location' in response.headers, \
        f'No Location header when requesting HEAD on "{url}"'

    location = response.headers['location']
    return os.path.basename(location)


def _get_current_slot(opener, rpc_url):
    request = urllib.request.Request(
        headers={'Content-Type': 'application/json'},
        data=b'{"jsonrpc":"2.0", "id":1, "method":"getSlot"}',
        url=rpc_url,
        method='POST',
    )
    response = opener.open(request, timeout=_HTTP_TIMEOUT)

    assert response.status == 200, \
        f'Wrong status when requesting POST on "{rpc_url}": {response.status} (should be 200)'

    response = response.read()
    response = json.loads(response)
    return response['result']


def _create_opener():
    opener = urllib.request.OpenerDirector()
    opener.add_handler(urllib.request.HTTPHandler())
    return opener


def _read_destaked(path):
    with open(path, 'r') as destaked_f:
        vote_accounts = set()
        for line in destaked_f:
            line = line.strip()
            if not line.startswith('--destake-vote-account'):
                continue
            line = line.split()
            assert len(line) >= 2, f'Too few parts in destake line: {line}'
            assert len(line) <= 3, f'Too many parts in destake line: {line}'
            vote_accounts.add(line[1])
        return frozenset(vote_accounts)


def _read_validators(path):
    with open(path, 'r') as validators_f:
        validators_list = json.load(validators_f)

    total_active_stake = validators_list['totalActiveStake']
    validators_list = validators_list['validators']
    validators = dict()
    for validator in validators_list:
        val_id = validator['identityPubkey']
        val_vote = validator['voteAccountPubkey']
        val_stake = validator['activatedStake']
        assert val_stake >= 0, f'Unexpected activatedStake: {val_stake}'
        assert val_id not in validators, f'Duplicate validator ID: {val_id}'
        validators[val_id] = dict(pubkey=val_vote, stake=val_stake)
    return total_active_stake, validators


def _read_cluster_nodes(path):
    with open(path, 'r') as nodes_f:
        return json.load(nodes_f)['result']


_main()