Remove stale pools from blockchain.info's known pools.json

rmpools.py

"""Remove stale pool signatures.

Count the occurrences of pools.json signatures, and remove all
that have not been seen in <expiry> blocks.

pools.json is from: https://github.com/blockchain/Blockchain-Known-Pools

usage: python rmpools.py <expiry> <path to pools.json>

requirements: python2.7

              python-bitcoinlib
              https://pypi.python.org/pypi/python-bitcoinlib
"""
import sys
import json
import io

from bitcoin.rpc import Proxy
from bitcoin.wallet import CBitcoinAddress, CBitcoinAddressError


def get_coinbase_info(blockheight, proxy):
    '''Gets coinbase tag and addresses of a block with specified height.

    Returns:
        addresses - A list of p2sh/p2pkh addresses corresponding to the
                    outputs. Returns None in place of an unrecognizable
                    scriptPubKey.
        tag - the UTF-8 decoded scriptSig.
    '''
    block = proxy.getblock(proxy.getblockhash(blockheight))
    coinbase_tx = block.vtx[0]
    assert coinbase_tx.is_coinbase()
    addresses = []
    for output in coinbase_tx.vout:
        try:
            addr = str(CBitcoinAddress.from_scriptPubKey(output.scriptPubKey))
        except (CBitcoinAddressError, ValueError):
            addr = None
        else:
            addr = addr.decode('ascii')
        addresses.append(addr)

    tag = str(coinbase_tx.vin[0].scriptSig).decode('utf-8', 'ignore')

    return addresses, tag


def countknownpools(knownpools, expiry):
    """Count occurences of known pool signatures.

    knownpools is the dict representation of pools.json.
    expiry is the number of past blocks, with reference to the current
    blockheight, to include in the signature count.
    """
    proxy = Proxy()
    currheight = proxy.getblockcount()

    known_addr_count = {
        addr: 0 for addr in knownpools['payout_addresses']}
    known_tag_count = {
        tag: 0 for tag in knownpools['coinbase_tags']}

    for height in range(currheight-expiry, currheight+1):
        while True:
            try:
                block_addrs, block_tag = get_coinbase_info(height, proxy)
            except Exception:
                # In case of timeout error, try again.
                proxy = Proxy()
            else:
                break
        for known_tag in knownpools['coinbase_tags']:
            if known_tag in block_tag:
                known_tag_count[known_tag] += 1
        for known_addr in knownpools['payout_addresses']:
            if known_addr in block_addrs:
                known_addr_count[known_addr] += 1
        if not height % 100:
            print(height)

    return known_addr_count, known_tag_count


if __name__ == "__main__":
    with open(sys.argv[2], 'r') as f:
        knownpools = json.load(f)
    expiry = int(sys.argv[1])
    addr_count, tag_count = countknownpools(knownpools, expiry)

    print("Addresses unseen for {} blocks:".format(expiry))
    print("==============================")
    for addr, count in addr_count.items():
        if not count:
            print("{:40}: {}".format(
                knownpools['payout_addresses'][addr]['name'], addr))
            del knownpools['payout_addresses'][addr]
    print("")
    print("Tags unseen for {} blocks:".format(expiry))
    print("==========================")
    for tag, count in tag_count.items():
        if not count:
            print("{:40}: {}".format(
                knownpools['coinbase_tags'][tag]['name'], tag))
            del knownpools['coinbase_tags'][tag]

    with io.open("cleaned_pools.json", "w", encoding="utf8") as f:
        data = json.dumps(knownpools, f, indent=4, separators=(',', ' : '),
                          ensure_ascii=False)
        f.write(unicode(data))