create_bundle_from_mempool_items Simulation

create_bundle_from_mempool_items.py

import os
import json
import time
import sys


def parse_from_json_file(filename):
    with open(filename, "r") as f:
        data = json.load(f)
    return data["mempool_items"]


def create_fee_cost_array(items):
    fee_cost_array = []
    for key, item in items.items():
        fee = item["fee"]
        cost = item["cost"]
        fpc = fee / cost
        fee_cost_array.append({"key": key, "fee": fee, "cost": cost, "fpc": fpc})
    return sorted(fee_cost_array, key=lambda x: x["fpc"], reverse=True)


def find_items_until_max_cost(fee_cost_array, max_cost):
    print("\n\n******** Finding items until max cost ********")
    cumulative_cost = 0
    items_until_max_cost = []
    for item in fee_cost_array:
        print(
            f'Cumulative Cost: {cumulative_cost}, Cost: {item["cost"]}, FPC: {item["fpc"]}, percent: {(item["cost"]/max_cost)*100}, cummulative percent: {(cumulative_cost/max_cost)*100}'
        )
        if cumulative_cost + item["cost"] > max_cost:
            print(
                f'Breaking at {item["key"]}, cost would exceed max cost: {cumulative_cost+item["cost"]}'
            )
            break
        cumulative_cost += item["cost"]
        items_until_max_cost.append(item)

    print(
        f"Cumulative Cost: {cumulative_cost}, Proportion full: {cumulative_cost/max_cost}"
    )
    return (
        sum([item["fee"] for item in items_until_max_cost]),
        len(items_until_max_cost),
        cumulative_cost / max_cost,
    )


def find_items_until_max_cost_with_retry(fee_cost_array, max_cost):
    print("\nm******** Finding items until max cost with retry ********")
    retry_add_more_items = 0
    max_retry_add_more_items = 10
    
    nb_retry = 0
    current_max = 0
    total_skipped = 0
    space_left = max_cost
    MIN_COST_THRESHOLD = 6_000_000

    cumulative_cost = 0
    items_until_max_cost = []
    for item in fee_cost_array:
        print(
            f'Cumulative Cost: {cumulative_cost}, Cost: {item["cost"]}, FPC: {item["fpc"]}, percent: {(item["cost"]/max_cost)*100}, cummulative percent: {(cumulative_cost/max_cost)*100}'
        )
        if cumulative_cost + item["cost"] > max_cost:
            print(
                f'Cost sum exceeds max block cost: {cumulative_cost+item["cost"]}, cumulative percentage: {((cumulative_cost + item["cost"]) / max_cost)*100}'
            )
            if retry_add_more_items >= max_retry_add_more_items:
                break
            retry_add_more_items += 1
            total_skipped += 1
            print(
                f"Retry to add more items: {retry_add_more_items}/{max_retry_add_more_items}"
            )
            continue
        if retry_add_more_items > 0:
            nb_retry += 1

        current_max = max([current_max, retry_add_more_items])
        cumulative_cost += item["cost"]
        items_until_max_cost.append(item)
        space_left -= item["cost"]
        retry_add_more_items = 0
        if space_left < MIN_COST_THRESHOLD:
            print(f"Space left is less than {MIN_COST_THRESHOLD}, stopping")
            break

    print(
        f"Cumulative Cost: {cumulative_cost}, Proportion full: {cumulative_cost/max_cost}"
    )

    return (
        sum([item["fee"] for item in items_until_max_cost]),
        len(items_until_max_cost),
        cumulative_cost / max_cost,
        nb_retry,
        current_max,
        total_skipped
    )


def do_test(file_path):
    print(f"********** Processing file: {file_path} **********")
    items = parse_from_json_file(file_path)
    fee_cost_array = create_fee_cost_array(items)

    time_start = time.perf_counter()
    fees1, length1, cumulative_percent1 = find_items_until_max_cost(
        fee_cost_array, 5500000000
    )
    time_end = time.perf_counter()
    time1 = time_end - time_start

    time_start = time.perf_counter()
    (
        fees2,
        length2,
        cumulative_percent2,
        nb_retry,
        current_max,
        total_skipped
    ) = find_items_until_max_cost_with_retry(fee_cost_array, 5500000000)
    time_end = time.perf_counter()
    time2 = time_end - time_start

    print(f"\n********** File: {file_path} **********")
    print("***** Diff ******")
    print(f"Gain {fees2-fees1} mojos")
    print(f"{length2-length1} more tx were included")
    print(f"{nb_retry} retries were done")
    print(f"{total_skipped} tx were skipped")
    print(f"Current max retry is {current_max}")
    print(f"cumulative_percent1: {cumulative_percent1*100}")
    print(f"cumulative_percent2: {cumulative_percent2*100}")
    print(
        f"cumulative_percent2 is {cumulative_percent2/cumulative_percent1} times bigger"
    )

    print(f"find_items_until_max_cost: {time1} seconds")
    print(f"find_items_until_max_cost_with_retry: {time2} seconds")
    print(f"find_items_until_max_cost_with_retry is {time2/time1} times slower")
    print(f"find_items_until_max_cost_with_retry took {time2-time1} seconds more")
    print("***********\n\n")


def process_files_in_directory(directory):
    for filename in os.listdir(directory):
        if filename.endswith(".json"):  # process only JSON files
            file_path = os.path.join(directory, filename)
            do_test(file_path)


process_files_in_directory("mempool_items")

results_10_small.txt

********** File: mempool_items/all_mempool_items_24_01_22_00_49.json **********
***** Diff ******
Gain 0 mojos
0 more tx were included
0 retries were done
0 tx were skipped
Current max retry is 0
cumulative_percent1: 99.9519226
cumulative_percent2: 99.9519226
cumulative_percent2 is 1.0 times bigger
find_items_until_max_cost: 0.0009719999507069588 seconds
find_items_until_max_cost_with_retry: 0.0008937920210883021 seconds
find_items_until_max_cost_with_retry is 0.9195391629785844 times slower
find_items_until_max_cost_with_retry took -7.820792961865664e-05 seconds more
***********


********** File: mempool_items/example_mempool_17_10_22.json **********
***** Diff ******
Gain 0 mojos
12 more tx were included
8 retries were done
23 tx were skipped
Current max retry is 3
cumulative_percent1: 64.72220256363637
cumulative_percent2: 99.80760176363637
cumulative_percent2 is 1.542092169460754 times bigger
find_items_until_max_cost: 4.9291993491351604e-05 seconds
find_items_until_max_cost_with_retry: 0.00015441689174622297 seconds
find_items_until_max_cost_with_retry is 3.132697235572665 times slower
find_items_until_max_cost_with_retry took 0.00010512489825487137 seconds more
***********


********** File: mempool_items/all_mempool_items_16_01_24_16_07.json **********
***** Diff ******
Gain 11374116 mojos
105 more tx were included
2 retries were done
12 tx were skipped
Current max retry is 1
cumulative_percent1: 47.63903032727273
cumulative_percent2: 99.83074730909091
cumulative_percent2 is 2.09556631659522 times bigger
find_items_until_max_cost: 0.0005877499934285879 seconds
find_items_until_max_cost_with_retry: 0.0007545830449089408 seconds
find_items_until_max_cost_with_retry is 1.2838503672405794 times slower
find_items_until_max_cost_with_retry took 0.00016683305148035288 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_16_00.json **********
***** Diff ******
Gain 0 mojos
0 more tx were included
0 retries were done
0 tx were skipped
Current max retry is 0
cumulative_percent1: 99.89546810909091
cumulative_percent2: 99.89546810909091
cumulative_percent2 is 1.0 times bigger
find_items_until_max_cost: 0.0011688750237226486 seconds
find_items_until_max_cost_with_retry: 0.0011736250016838312 seconds
find_items_until_max_cost_with_retry is 1.0040637175615703 times slower
find_items_until_max_cost_with_retry took 4.749977961182594e-06 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_08_13.json **********
***** Diff ******
Gain 7856698 mojos
227 more tx were included
1 retries were done
11 tx were skipped
Current max retry is 1
cumulative_percent1: 17.909095545454544
cumulative_percent2: 99.85378789090909
cumulative_percent2 is 5.575590773831831 times bigger
find_items_until_max_cost: 0.00022087490651756525 seconds
find_items_until_max_cost_with_retry: 0.0007111660670489073 seconds
find_items_until_max_cost_with_retry is 3.2197685027309846 times slower
find_items_until_max_cost_with_retry took 0.000490291160531342 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_00_28.json **********
***** Diff ******
Gain 23089406 mojos
113 more tx were included
2 retries were done
7 tx were skipped
Current max retry is 6
cumulative_percent1: 28.7863362
cumulative_percent2: 99.94450874545454
cumulative_percent2 is 3.4719426623473724 times bigger
find_items_until_max_cost: 0.00026929203886538744 seconds
find_items_until_max_cost_with_retry: 0.0005285420920699835 seconds
find_items_until_max_cost_with_retry is 1.9627096823838481 times slower
find_items_until_max_cost_with_retry took 0.00025925005320459604 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_08_09.json **********
***** Diff ******
Gain 0 mojos
0 more tx were included
0 retries were done
0 tx were skipped
Current max retry is 0
cumulative_percent1: 99.9459028
cumulative_percent2: 99.9459028
cumulative_percent2 is 1.0 times bigger
find_items_until_max_cost: 0.0011033749906346202 seconds
find_items_until_max_cost_with_retry: 0.0011169170029461384 seconds
find_items_until_max_cost_with_retry is 1.0122732637829042 times slower
find_items_until_max_cost_with_retry took 1.3542012311518192e-05 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_10_31.json **********
***** Diff ******
Gain 0 mojos
0 more tx were included
0 retries were done
0 tx were skipped
Current max retry is 0
cumulative_percent1: 99.93169045454545
cumulative_percent2: 99.93169045454545
cumulative_percent2 is 1.0 times bigger
find_items_until_max_cost: 0.0011591659858822823 seconds
find_items_until_max_cost_with_retry: 0.0012181249912828207 seconds
find_items_until_max_cost_with_retry is 1.050863298370218 times slower
find_items_until_max_cost_with_retry took 5.8959005400538445e-05 seconds more
***********


********** File: mempool_items/all_mempool_items_16_01_24_16_49.json **********
***** Diff ******
Gain 10633580 mojos
48 more tx were included
2 retries were done
12 tx were skipped
Current max retry is 1
cumulative_percent1: 65.64904625454545
cumulative_percent2: 99.85798996363636
cumulative_percent2 is 1.5210882055536692 times bigger
find_items_until_max_cost: 0.0005249169189482927 seconds
find_items_until_max_cost_with_retry: 0.0006302079418674111 seconds
find_items_until_max_cost_with_retry is 1.2005860720398882 times slower
find_items_until_max_cost_with_retry took 0.0001052910229191184 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_00_30.json **********
***** Diff ******
Gain 0 mojos
0 more tx were included
0 retries were done
0 tx were skipped
Current max retry is 0
cumulative_percent1: 99.94895058181818
cumulative_percent2: 99.94895058181818
cumulative_percent2 is 1.0 times bigger
find_items_until_max_cost: 0.00066708296071738 seconds
find_items_until_max_cost_with_retry: 0.0006381249986588955 seconds
find_items_until_max_cost_with_retry is 0.9565901637971037 times slower
find_items_until_max_cost_with_retry took -2.8957962058484554e-05 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_10_26.json **********
***** Diff ******
Gain 8700920 mojos
84 more tx were included
1 retries were done
1 tx were skipped
Current max retry is 1
cumulative_percent1: 28.66557518181818
cumulative_percent2: 99.97610669090909
cumulative_percent2 is 3.487671398769675 times bigger
find_items_until_max_cost: 0.0003363749710842967 seconds
find_items_until_max_cost_with_retry: 0.00044475006870925426 seconds
find_items_until_max_cost_with_retry is 1.3221853792379767 times slower
find_items_until_max_cost_with_retry took 0.00010837509762495756 seconds more
***********


********** File: mempool_items/all_mempool_items_16_01_24_16_32.json **********
***** Diff ******
Gain 11793065 mojos
114 more tx were included
2 retries were done
12 tx were skipped
Current max retry is 1
cumulative_percent1: 43.58292434545454
cumulative_percent2: 99.86921043636363
cumulative_percent2 is 2.291475662458144 times bigger
find_items_until_max_cost: 0.00034187501296401024 seconds
find_items_until_max_cost_with_retry: 0.000581375090405345 seconds
find_items_until_max_cost_with_retry is 1.7005486460238828 times slower
find_items_until_max_cost_with_retry took 0.00023950007744133472 seconds more
***********


********** File: mempool_items/all_mempool_items_16_01_24_19_46.json **********
***** Diff ******
Gain 10633580 mojos
48 more tx were included
2 retries were done
12 tx were skipped
Current max retry is 1
cumulative_percent1: 65.64904625454545
cumulative_percent2: 99.85798996363636
cumulative_percent2 is 1.5210882055536692 times bigger
find_items_until_max_cost: 0.0005193749675527215 seconds
find_items_until_max_cost_with_retry: 0.000620416016317904 seconds
find_items_until_max_cost_with_retry is 1.194543547682486 times slower
find_items_until_max_cost_with_retry took 0.0001010410487651825 seconds more
***********


********** File: mempool_items/all_mempool_items_24_01_17_16_25.json **********
***** Diff ******
Gain 5472283 mojos
479 more tx were included
1 retries were done
1 tx were skipped
Current max retry is 1
cumulative_percent1: 17.62220050909091
cumulative_percent2: 99.9927582
cumulative_percent2 is 5.674249260097564 times bigger
find_items_until_max_cost: 0.00019720802083611488 seconds
find_items_until_max_cost_with_retry: 0.0011561670107766986 seconds
find_items_until_max_cost_with_retry is 5.862677419888029 times slower
find_items_until_max_cost_with_retry took 0.0009589589899405837 seconds more
***********