Crawl Ethereum Transaction

crawl_eth_transaction.py

import time
import concurrent.futures
from collections import namedtuple

latest_height = 6002659
safe_height = latest_height - 30
begin_height = safe_height - 1000

Eth_transaction = namedtuple('Eth_transactions', ['from_address', 'to_address', 'amount'])

def get_receipt(tx):
    if isinstance(tx, AttributeDict):
        tx = tx.hash
    return web3.eth.getTransactionReceipt(tx)

def crawler(height):
    # Get transactions from block
    block = web3.eth.getBlock(height, full_transactions=True)
    
    # Get receipt of each transaction
    receipts = []
    with concurrent.futures.ThreadPoolExecutor() as executor:
        for receipt in executor.map(get_receipt, block.transactions):
            receipts.append(receipt)
    
    # Parse the transactions
    if len(receipts) != len(block.transactions):
        raise Exception("Receipt count not equal")

    if any(receipt is None for receipt in receipts):
        raise Exception("some receipt is None")
    
    
    tx_status = {
        receipt.transactionHash: receipt.status 
        for receipt in receipts
    }
    
    eth_transactions = []
    for tx in block.transactions:
        if tx.value <= 0:
            continue
        if tx.hash not in tx_status:
            raise Exception("Tx not in recripts error")
        if tx_status[tx.hash] != 1:
            continue

        eth_transactions.append(Eth_transaction(
                from_address=tx['from'], 
                to_address=tx['to'],
                amount=web3.fromWei(tx['value'], 'ether'),
        ))
    return eth_transactions

    
eth_transactions = []
start_at = time.time()
for height in range(begin_height, safe_height):
    eth_transactions += crawler(height)
print(f'Crawel from {begin_height} to {safe_height}, total {len(eth_transactions)} transactions')
print(f'Crawling Time: {time.time() - start_at} s')

''' Output
Crawel from 6001629 to 6002629, total 68328 transactions
Crawling Time: 3552.4722516536713 s
'''