slowest py-evm tests

0-get-duration-samples.sh

#!/bin/sh
# extracted from shell history (i.e. not actual script)

# spoof "run slowest, too"
export TRAVIS_EVENT_TYPE=cron

# run 10 times, obtain N slowest each time
for r in `seq 1 10`; do
    echo "===== starting run $r =====" >> durations
    pytest --quiet --durations=10000 --numprocesses=8 tests/json-fixtures/test_blockchain.py | tee -a durations
    sleep 1
done

# sort by duration
cat durations | grep call | sort -nr > durations-sorted

# reorder columns
awk ' { t = $1; $1 = $3; $2 = t; $3 =""; print; } ' durations-sorted > durations-actual.txt

# blockchain_fixture_mark_fn() excludes some manually, so have to do it here, too :/
cat durations-actual.txt | \
    grep -v 'bcExploitTest' | \
    grep -v 'bcForkStressTest/ForkStressTest.json' | \
    grep -v 'bcWalletTest/walletReorganizeOwners.json' > durations.txt

1-slowest-tests.py

#!/usr/bin/env python

import csv
from decimal import Decimal as D
import re

slowest = dict()
with open('durations.txt', 'r') as durfile:
    reader = csv.reader(durfile, delimiter=' ')
    for row in reader:
        testname = row[0]
        testdur = D(row[1][:-1]) # chomp off trailing 's' ("seconds")
        if testname not in slowest.keys():
            slowest[testname] = D('0')
        slowest[testname] += testdur

# sort by value (reversing key->value to value->key)
# https://stackoverflow.com/a/11229116/7056908
slowest_sorted_by_durations = sorted(
    ((v,k) for k,v in slowest.items()),
    reverse=True
)

# truncate
slowest_100_list_of_tuples = slowest_sorted_by_durations[:100]
# discard durations
slowest_100_list = [el[1] for el in slowest_100_list_of_tuples]
# sort alphabetically
slowest_100 = sorted(slowest_100_list)

print('SLOWEST_TESTS = {')
for testname in slowest_100:
    match = re.match(r'.*\[.*BlockchainTests\/(.*):(.*):.*\]', testname)
    fullpath = match.group(1)
    ident = match.group(2)
    print(f"    ('{fullpath}', '{ident}'),  # noqa: E501")
print('}')

slowest-tests-output.py

SLOWEST_TESTS = {
    ('GeneralStateTests/stAttackTest/ContractCreationSpam.json', 'ContractCreationSpam_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCallCreateCallCodeTest/Call1024BalanceTooLow.json', 'Call1024BalanceTooLow_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCallCreateCallCodeTest/Call1024PreCalls.json', 'Call1024PreCalls_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCallCreateCallCodeTest/Call1024PreCalls.json', 'Call1024PreCalls_d0g1v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCallCreateCallCodeTest/CallRecursiveBombPreCall.json', 'CallRecursiveBombPreCall_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCallCreateCallCodeTest/Callcode1024BalanceTooLow.json', 'Callcode1024BalanceTooLow_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stChangedEIP150/Call1024BalanceTooLow.json', 'Call1024BalanceTooLow_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stChangedEIP150/Call1024PreCalls.json', 'Call1024PreCalls_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stChangedEIP150/Call1024PreCalls.json', 'Call1024PreCalls_d0g1v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stChangedEIP150/Callcode1024BalanceTooLow.json', 'Callcode1024BalanceTooLow_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCreate2/Create2OnDepth1024.json', 'Create2OnDepth1024_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCreate2/Create2Recursive.json', 'Create2Recursive_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stCreate2/Create2Recursive.json', 'Create2Recursive_d0g1v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stDelegatecallTestHomestead/Call1024BalanceTooLow.json', 'Call1024BalanceTooLow_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stDelegatecallTestHomestead/Call1024PreCalls.json', 'Call1024PreCalls_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stDelegatecallTestHomestead/Call1024PreCalls.json', 'Call1024PreCalls_d0g1v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stDelegatecallTestHomestead/Delegatecall1024.json', 'Delegatecall1024_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRandom/randomStatetest48.json', 'randomStatetest48_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRandom2/randomStatetest458.json', 'randomStatetest458_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRandom2/randomStatetest467.json', 'randomStatetest467_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRandom2/randomStatetest636.json', 'randomStatetest636_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRandom2/randomStatetest639.json', 'randomStatetest639_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRecursiveCreate/recursiveCreateReturnValue.json', 'recursiveCreateReturnValue_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRevertTest/LoopCallsDepthThenRevert2.json', 'LoopCallsDepthThenRevert2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRevertTest/LoopCallsDepthThenRevert3.json', 'LoopCallsDepthThenRevert3_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRevertTest/LoopCallsThenRevert.json', 'LoopCallsThenRevert_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stRevertTest/LoopCallsThenRevert.json', 'LoopCallsThenRevert_d0g1v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stShift/shiftCombinations.json', 'shiftCombinations_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1024BalanceTooLow.json', 'static_Call1024BalanceTooLow_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1024BalanceTooLow2.json', 'static_Call1024BalanceTooLow2_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1024PreCalls.json', 'static_Call1024PreCalls_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1024PreCalls2.json', 'static_Call1024PreCalls2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1024PreCalls2.json', 'static_Call1024PreCalls2_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1024PreCalls3.json', 'static_Call1024PreCalls3_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth.json', 'static_Call1MB1024Calldepth_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000.json', 'static_Call50000_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000.json', 'static_Call50000_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_ecrec.json', 'static_Call50000_ecrec_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_ecrec.json', 'static_Call50000_ecrec_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_identity.json', 'static_Call50000_identity_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_identity.json', 'static_Call50000_identity_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_identity2.json', 'static_Call50000_identity2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_identity2.json', 'static_Call50000_identity2_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_rip160.json', 'static_Call50000_rip160_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000_rip160.json', 'static_Call50000_rip160_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1.json', 'static_Call50000bytesContract50_1_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_1.json', 'static_Call50000bytesContract50_1_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2.json', 'static_Call50000bytesContract50_2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Call50000bytesContract50_2.json', 'static_Call50000bytesContract50_2_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_LoopCallsDepthThenRevert2.json', 'static_LoopCallsDepthThenRevert2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_LoopCallsDepthThenRevert3.json', 'static_LoopCallsDepthThenRevert3_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_LoopCallsThenRevert.json', 'static_LoopCallsThenRevert_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_LoopCallsThenRevert.json', 'static_LoopCallsThenRevert_d0g1v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stStaticCall/static_Return50000_2.json', 'static_Return50000_2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stSystemOperationsTest/CallRecursiveBomb0_OOG_atMaxCallDepth.json', 'CallRecursiveBomb0_OOG_atMaxCallDepth_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stTimeConsuming/CALLBlake2f_MaxRounds.json', 'CALLBlake2f_MaxRounds_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stTimeConsuming/static_Call50000_sha256.json', 'static_Call50000_sha256_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stTimeConsuming/static_Call50000_sha256.json', 'static_Call50000_sha256_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_one_point_fail.json', 'ecpairing_one_point_fail_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_three_point_fail_1.json', 'ecpairing_three_point_fail_1_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_three_point_match_1.json', 'ecpairing_three_point_match_1_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_three_point_match_1.json', 'ecpairing_three_point_match_1_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_fail_1.json', 'ecpairing_two_point_fail_1_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_fail_2.json', 'ecpairing_two_point_fail_2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_1.json', 'ecpairing_two_point_match_1_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_2.json', 'ecpairing_two_point_match_2_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_2.json', 'ecpairing_two_point_match_2_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_3.json', 'ecpairing_two_point_match_3_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_3.json', 'ecpairing_two_point_match_3_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_4.json', 'ecpairing_two_point_match_4_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_match_4.json', 'ecpairing_two_point_match_4_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_oog.json', 'ecpairing_two_point_oog_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_point_oog.json', 'ecpairing_two_point_oog_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_points_with_one_g2_zero.json', 'ecpairing_two_points_with_one_g2_zero_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/ecpairing_two_points_with_one_g2_zero.json', 'ecpairing_two_points_with_one_g2_zero_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d0g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d0g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d1g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d1g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d2g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d2g3v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d3g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d4g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d5g0v0_Istanbul'),  # noqa: E501
    ('GeneralStateTests/stZeroKnowledge/pairingTest.json', 'pairingTest_d5g3v0_Istanbul'),  # noqa: E501
    ('InvalidBlocks/bcForgedTest/bcForkBlockTest.json', 'BlockWrongResetGas'),  # noqa: E501
    ('InvalidBlocks/bcForgedTest/bcInvalidRLPTest.json', 'BLOCK_difficulty_TooLarge'),  # noqa: E501
    ('InvalidBlocks/bcMultiChainTest/UncleFromSideChain.json', 'UncleFromSideChain_Constantinople'),  # noqa: E501
    ('TransitionTests/bcHomesteadToDao/DaoTransactions.json', 'DaoTransactions'),  # noqa: E501
    ('TransitionTests/bcHomesteadToDao/DaoTransactions_UncleExtradata.json', 'DaoTransactions_UncleExtradata'),  # noqa: E501
    ('ValidBlocks/bcGasPricerTest/RPC_API_Test.json', 'RPC_API_Test_EIP150'),  # noqa: E501
    ('ValidBlocks/bcGasPricerTest/RPC_API_Test.json', 'RPC_API_Test_EIP158'),  # noqa: E501
    ('ValidBlocks/bcGasPricerTest/RPC_API_Test.json', 'RPC_API_Test_Frontier'),  # noqa: E501
    ('ValidBlocks/bcGasPricerTest/RPC_API_Test.json', 'RPC_API_Test_Homestead'),  # noqa: E501
    ('ValidBlocks/bcRandomBlockhashTest/randomStatetest284BC.json', 'randomStatetest284BC_Byzantium'),  # noqa: E501
    ('ValidBlocks/bcStateTests/randomStatetest94.json', 'randomStatetest94_Byzantium'),  # noqa: E501
    ('ValidBlocks/bcStateTests/randomStatetest94.json', 'randomStatetest94_Constantinople'),  # noqa: E501
    ('ValidBlocks/bcStateTests/randomStatetest94.json', 'randomStatetest94_ConstantinopleFix'),  # noqa: E501
    ('ValidBlocks/bcStateTests/randomStatetest94.json', 'randomStatetest94_Homestead'),  # noqa: E501
    ('ValidBlocks/bcStateTests/randomStatetest94.json', 'randomStatetest94_Istanbul'),  # noqa: E501
}