Profiling with yummy-spam

bigmempool.py

#!/usr/bin/env python3
# Copyright (c) 2024 Patrick Lodder

"""
big mempool test
1. Get 8 utxo
2. Split into 500k utxo
3. fill up mempool
6. run mempool queries as we go
""" 

import os
import subprocess

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
from test_framework.mininode import CTransaction, CTxIn, CTxOut, COutPoint, wait_until
from test_framework.script import CScript

def start_valgrind_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None):
    """
    Start a dogecoind with valgrind and return an RPC connection to it
    """
    datadir = os.path.join(dirname, "node"+str(i))
    if binary is None:
        binary = os.getenv("DOGECOIND", "dogecoind")

    args = [ "valgrind", "--tool=callgrind", "--dump-instr=yes", "--simulate-cache=yes", "--collect-jumps=yes", "--quiet", "--collect-atstart=no", "--instr-atstart=no" ]
    dogecoind_args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-mocktime="+str(get_mocktime()) ]
    args.extend(dogecoind_args)
    
    if extra_args is not None: args.extend(extra_args)
    bitcoind_processes[i] = subprocess.Popen(args)
    print("start_valgrind_node: dogecoind started with valgrind, waiting for RPC to come up")
    url = rpc_url(i, rpchost)
    wait_for_bitcoind_start(bitcoind_processes[i], url, i)
    print("start_valgrind_node: RPC successfully started")
    proxy = get_rpc_proxy(url, i, timeout=timewait)
    
    return proxy

class BigMempoolTest(BitcoinTestFramework):

    def __init__(self):
        super().__init__()
        self.setup_clean_chain = True
        self.num_nodes = 2

    def setup_network(self):
        self.nodes = []
        self.nodes.append(start_node(0, self.options.tmpdir))
        self.nodes.append(start_valgrind_node(1, self.options.tmpdir, timewait=300))
        
        self.is_network_split = True

    def sync_mempool_by_info(self, timeout=30):
        def compare_mempool_info():
            first_result=self.nodes[0].getmempoolinfo()["size"]
            for node in self.nodes[1:]:
                if node.getmempoolinfo()["size"] != first_result:
                    return False
            return True
        wait_until(compare_mempool_info, timeout=timeout) 

    def split_utxo(self, nodeidx, input, num, fee, target):
        assert fee < input['amount']
        amount = input['amount'] - fee
        amount_each = satoshi_round(amount / num)
        amount_sats = int(amount_each * 100000000)

        tx = CTransaction()
        tx.vin.append(CTxIn(COutPoint(int(input['txid'], 16), input['vout'])))
        bytes_target = hex_str_to_bytes(target)
        
        for i in range(0,num):
            tx.vout.append(CTxOut(amount_sats, CScript(bytes_target)))

        compiled = tx.serialize_without_witness()
        signedtx = self.nodes[nodeidx].signrawtransaction(bytes_to_hex_str(compiled))

        self.nodes[nodeidx].sendrawtransaction(signedtx['hex'])

    def spend_utxos_to_dust(self, nodeidx, inputs, fee, target1, target2):
        amount = Decimal("0") - fee

        tx = CTransaction()
        
        for input in inputs:
            amount += input['amount']
            tx.vin.append(CTxIn(COutPoint(int(input['txid'], 16), input['vout'])))
        assert amount > 0.01

        dust_output = 100000
        change = int(amount * 100000000) - dust_output

        tx.vout.append(CTxOut(dust_output, CScript(hex_str_to_bytes(target1))))
        tx.vout.append(CTxOut(dust_output, CScript(hex_str_to_bytes(target2))))
        
        compiled = tx.serialize_without_witness()
        signedtx = self.nodes[nodeidx].signrawtransaction(bytes_to_hex_str(compiled))

        self.nodes[nodeidx].sendrawtransaction(signedtx['hex'])

    def run_test(self):
        # mine 68 blocks to get 8 spendable utxo
        self.nodes[0].generate(68)
        
        utxos = self.nodes[0].listunspent()
        assert len(utxos) == 8

        addr = self.nodes[0].getnewaddress()
        target = self.nodes[0].validateaddress(addr)['scriptPubKey']

        # split 8 utxo into 2000 utxo
        print("... splitting 8->2000")
        for utxo in utxos:
            self.split_utxo(0, utxo, 250, 5, target)

        print("... mining 1 block")
        self.nodes[0].generate(1)
        
        # get all mined tx with less than 10 confirmations (everything we just mined)
        utxos = self.nodes[0].listunspent(1,10)
        assert len(utxos) == 2000

        # split 2000 utxo into 500k utxo
        print("... splitting 2k->500k")
        for utxo in utxos:
            self.split_utxo(0, utxo, 250, 5, target)

        # each tx is under 10kB so we can fit at least 75 in each block
        # 2000 / 75 = 26.6
        print("... mining 27 blocks")
        self.nodes[0].generate(27)

        # now we have > 500k utxo on our chain
        num_utxo = self.nodes[0].gettxoutsetinfo()['txouts']
        assert num_utxo > 500000

        print("... connecting node 1")
        connect_nodes_bi(self.nodes, 0, 1)
        
        print("  ... synchronizing chain")
        self.is_network_split = False
        sync_blocks(self.nodes, wait=300)

        print("... Hammer time")

        #spam 500 x 500tx, slowly cuz we wanna callgrind
        addr2 = self.nodes[0].getnewaddress()
        target2 = self.nodes[0].validateaddress(addr2)['scriptPubKey']
        
        for j in range(0,500):
            for i in range(0,500):
                utxos = self.nodes[0].listunspent(1, 10, [], False, {"maximumCount": 2, "minimumAmount": 1})
                self.spend_utxos_to_dust(0, utxos, Decimal("0.2"), target, target2)

            try:
                self.nodes[1].getrawmempool()
                print("  ... successfully queried non-verbose getrawmempool")
            except Exception as e:
                print(f'  ... error querying non-verbose getrawmempool: {e}')

            try:
                self.nodes[1].getrawmempool(True)
                print("  ... successfully queried verbose getrawmempool")
            except Exception as e:
                print(f'  ... error querying verbose getrawmempool: {e}')

            retries = 10
            while retries > 0:
                try:
                    self.sync_mempool_by_info(30)
                    break
                except Exception as e:
                    print(f"  ... failed to sync mempool, retrying ({e})")
                    retries = retries - 1
                
            stats = self.nodes[1].getmempoolinfo()
            print(f'  ... mempool txs: {stats["size"]}; bytes: {stats["bytes"]}')

if __name__ == '__main__':
    BigMempoolTest().main()

inline-callgrind-getrawmempool.patch

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index e6bccb90e7..2425549409 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -27,6 +27,7 @@
 #include <univalue.h>
 
 #include <boost/thread/thread.hpp> // boost::thread::interrupt
+#include <valgrind/callgrind.h>
 
 #include <mutex>
 #include <condition_variable>
@@ -477,11 +478,18 @@ UniValue getrawmempool(const JSONRPCRequest& request)
             + HelpExampleRpc("getrawmempool", "true")
         );
 
+    CALLGRIND_START_INSTRUMENTATION;
+    CALLGRIND_TOGGLE_COLLECT;
+
     bool fVerbose = false;
     if (request.params.size() > 0)
         fVerbose = request.params[0].get_bool();
 
-    return mempoolToJSON(fVerbose);
+    auto ret = mempoolToJSON(fVerbose);
+    CALLGRIND_TOGGLE_COLLECT;
+    CALLGRIND_STOP_INSTRUMENTATION;
+
+    return ret;
 }
 
 UniValue getmempoolancestors(const JSONRPCRequest& request)