solana-restart-failure-investigation-2022.11.06/leader-schedule

check-leader-schedule.py

#!/usr/bin/python3

import json
import math


_SNAP_SLOT = 160991176
_RIGHT_SHRED_VERSION = 4711

# wget -O destaked.txt 'https://gist.githubusercontent.com/CrazySerGo/9f14a3d4d5d2efca132dc9433b16a03e/raw/5e31452a40c1215f91d1ad962047e2f69c0b5731/Exclusion%2520list%252004.11.2022'
_DESTAKED_DUMP_PATH = 'destaked.txt'
# solana -u ${YOUR_RPC} --output json leader-schedule >leader-schedule.json
_LEADER_SCHEDULE_DUMP_PATH = 'leader-schedule.json'
# solana -u ${YOUR_RPC} --output json validators >validators.json
_VALIDATORS_DUMP_PATH = 'validators.json'
# curl ${YOUR_RPC} -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getClusterNodes"}' >cluster-nodes.json
_CLUSTER_NODES_DUMP_PATH = 'cluster-nodes.json'

_OUT_IMAGE_1_PATH = f'leader-schedule-after-{_SNAP_SLOT}.ppm'
_IMAGE_2_WIDTH = 1000
_OUT_IMAGE_2_PATH = f'leader-schedule-after-{_SNAP_SLOT}-first-{_IMAGE_2_WIDTH}.ppm'


def _main():
    destaked_vote_accs = _read_destaked(_DESTAKED_DUMP_PATH)
    leaders = _read_leader_schedule(_LEADER_SCHEDULE_DUMP_PATH)
    validator_vote_accs = _read_validators(_VALIDATORS_DUMP_PATH)
    nodes = _read_cluster_nodes(_CLUSTER_NODES_DUMP_PATH)

    print(f'Destaked validators: {len(destaked_vote_accs)} ({len(validator_vote_accs)} remaining)')

    destaked_or_offline_slots = 0
    slots_after_snapshot = 0
    destaked_or_offline_slots_after_snapshot = 0
    prev_slot = leaders[0]['slot'] - 1
    image = []
    for leader in leaders:
        leader_slot = leader['slot']
        # Sanity check.
        assert leader_slot == prev_slot + 1, f'Schedule gap before slot {leader_slot}'
        prev_slot = leader_slot

        leader_id = leader['leader']
        if (leader_id not in nodes) \
                or (leader_id not in validator_vote_accs) \
                or (validator_vote_accs[leader_id] in destaked_vote_accs):
            destaked_or_offline_slots += 1
            if leader_slot > _SNAP_SLOT:
                destaked_or_offline_slots_after_snapshot += 1
                image.append(False)
        else:
            if leader_slot > _SNAP_SLOT:
                image.append(True)

        if leader_slot > _SNAP_SLOT:
            slots_after_snapshot += 1

    print(f'Slots without leader:'
          f' {destaked_or_offline_slots}/{len(leaders)}, {destaked_or_offline_slots / len(leaders) * 100.0:.02f}%')
    print(f'Slots without leader after slot {_SNAP_SLOT}:'
          f' {destaked_or_offline_slots_after_snapshot}/{slots_after_snapshot},'
          f' {destaked_or_offline_slots_after_snapshot / slots_after_snapshot * 100.0:.02f}%')

    # Sanity check.
    assert len(image) == slots_after_snapshot, f'Wrong image width: {slots_after_snapshot}'

    with open(_OUT_IMAGE_1_PATH, 'w') as image_f:
        image_f.write('P3\n')

        width = len(image) // 255
        height = width // 8
        image_f.write(f'{width} {height}\n')
        image_f.write('255\n')

        for _y in range(height):
            for x in range(width):
                pixel_bool_range = image[x * 255:x * 255 + 255]
                assert len(pixel_bool_range) == 255
                redness = pixel_bool_range.count(True) / 255.0 * math.pi / 2.0
                red = int(math.sin(redness) * 255.0)
                blue = int(math.cos(redness) * 255.0)
                image_f.write(f'{red} 0 {blue}\n')

    with open(_OUT_IMAGE_2_PATH, 'w') as image_f:
        image_f.write('P3\n')

        height = _IMAGE_2_WIDTH // 8
        image_f.write(f'{_IMAGE_2_WIDTH} {height}\n')
        image_f.write('255\n')

        for _y in range(height):
            for pixel_bool in image[:_IMAGE_2_WIDTH]:
                if pixel_bool:
                    image_f.write('0 0 255\n')
                else:
                    image_f.write('255 0 0\n')


def _read_destaked(path):
    with open(path, 'r') as destaked_f:
        vote_accounts = set()
        for line in destaked_f:
            line = line.strip()
            if not line.startswith('--destake-vote-account'):
                continue
            line = line.split()
            assert len(line) >= 2, f'Too few parts in destake line: {line}'
            assert len(line) <= 3, f'Too many parts in destake line: {line}'
            vote_accounts.add(line[1])
        return frozenset(vote_accounts)


def _read_leader_schedule(path):
    with open(path, 'r') as schedule_f:
        schedule = json.load(schedule_f)

    print(f"Epoch: {schedule['epoch']}")
    return schedule['leaderScheduleEntries']


def _read_validators(path):
    with open(path, 'r') as validators_f:
        validators_list = json.load(validators_f)

    validators_list = validators_list['validators']
    validators = dict()
    for validator in validators_list:
        val_id = validator['identityPubkey']
        val_vote = validator['voteAccountPubkey']
        assert val_id not in validators, f'Duplicate validator ID: {val_id}'
        validators[val_id] = val_vote
    return validators


def _read_cluster_nodes(path):
    with open(path, 'r') as nodes_f:
        nodes = json.load(nodes_f)['result']
    return frozenset(node['pubkey'] for node in nodes if node['shredVersion'] == _RIGHT_SHRED_VERSION)


_main()