Comparing runtime of a for loop vs list comprehensions for a permutation test

spearman-perm.md

spearman-perm

Comparing runtime of a for loop vs comprehensions for a permutation test

Usage

$ python spearman-perm.py <p_value> <num_sims>

Results

$ python spearman-perm.py 0.05 100   
<function sig_v0 at 0x10c167ea0>	0.3029317855834961 seconds	 sig: 3
<function sig_v1 at 0x114a398c8>	0.4171440601348877 seconds	 sig: 7
<function sig_v2 at 0x114a39ae8>	0.2928578853607178 seconds	 sig: 4
<function sig_v3 at 0x114b89158>	0.3431227207183838 seconds	 sig: 7

$ python spearman-perm.py 0.05 1000
<function sig_v0 at 0x11044dea0>	4.248883962631226 seconds	 sig: 39
<function sig_v1 at 0x118d1e8c8>	4.142040014266968 seconds	 sig: 57
<function sig_v2 at 0x118d1eae8>	4.842396974563599 seconds	 sig: 70
<function sig_v3 at 0x118e6e158>	4.068037986755371 seconds	 sig: 52

$ python spearman-perm.py 0.05 1000
<function sig_v0 at 0x1073c5ea0>	2.671203136444092 seconds	 sig: 60
<function sig_v1 at 0x10fd618c8>	2.5390191078186035 seconds	 sig: 57
<function sig_v2 at 0x10fd61ae8>	2.7314538955688477 seconds	 sig: 52
<function sig_v3 at 0x10feb1158>	2.7248780727386475 seconds	 sig: 52

$ python spearman-perm.py 0.05 10000
<function sig_v0 at 0x108772ea0>	27.47206473350525 seconds	 sig: 551
<function sig_v1 at 0x1111148c8>	26.048126935958862 seconds	 sig: 555
<function sig_v2 at 0x111114ae8>	33.424891233444214 seconds	 sig: 563
<function sig_v3 at 0x111264158>	28.529727935791016 seconds	 sig: 534

v4: shuffle with for loop instead of permutation with comprehension

$  python spearman-perm.py 0.05 100
<function sig_v0 at 0x10f6752f0>	0.3031768798828125 seconds	 sig: 9
<function sig_v1 at 0x117ed59d8>	0.3173329830169678 seconds	 sig: 4
<function sig_v2 at 0x117ed5bf8>	0.31728315353393555 seconds	 sig: 4
<function sig_v3 at 0x118026268>	0.31329989433288574 seconds	 sig: 8
<function sig_v4 at 0x11802b2f0>	0.2999131679534912 seconds	 sig: 8

$ python spearman-perm.py 0.05 1000
<function sig_v0 at 0x1104912f0>	2.7923619747161865 seconds	 sig: 48
<function sig_v1 at 0x118cf09d8>	2.9415388107299805 seconds	 sig: 60
<function sig_v2 at 0x118cf0bf8>	2.852353096008301 seconds	 sig: 65
<function sig_v3 at 0x118e41268>	2.775151252746582 seconds	 sig: 49
<function sig_v4 at 0x118e452f0>	2.7235851287841797 seconds	 sig: 57

$ python spearman-perm.py 0.05 1000
<function sig_v0 at 0x106a162f0>	4.1564719676971436 seconds	 sig: 62
<function sig_v1 at 0x10f3419d8>	3.2942469120025635 seconds	 sig: 58
<function sig_v2 at 0x10f341bf8>	3.1608939170837402 seconds	 sig: 63
<function sig_v3 at 0x10f493268>	3.100147008895874 seconds	 sig: 66
<function sig_v4 at 0x10f4982f0>	3.0940558910369873 seconds	 sig: 37

$  python spearman-perm.py 0.05 10000
<function sig_v0 at 0x103c142f0>	32.10949993133545 seconds	 sig: 544
<function sig_v1 at 0x10c53e9d8>	30.17117691040039 seconds	 sig: 534
<function sig_v2 at 0x10c53ebf8>	43.16379380226135 seconds	 sig: 540
<function sig_v3 at 0x10c68f268>	37.967920780181885 seconds	 sig: 556
<function sig_v4 at 0x10c6932f0>	27.987082958221436 seconds	 sig: 547

$ python spearman-perm.py 0.05 10000
<function sig_v0 at 0x10b5f72f0>	35.95823407173157 seconds	 sig: 545
<function sig_v1 at 0x113f259d8>	30.673747062683105 seconds	 sig: 530
<function sig_v2 at 0x113f25bf8>	31.967529773712158 seconds	 sig: 552
<function sig_v3 at 0x114076268>	34.462607860565186 seconds	 sig: 544
<function sig_v4 at 0x11407b2f0>	29.492311000823975 seconds	 sig: 545

v5: shuffle with comprehension

$  python spearman-perm.py 0.05 1000
<function sig_v0 at 0x100cee2f0>	2.79156494140625 seconds	 sig: 42
<function sig_v1 at 0x10961d950>	2.776930093765259 seconds	 sig: 48
<function sig_v2 at 0x10961db70>	2.7937071323394775 seconds	 sig: 54
<function sig_v3 at 0x10976d1e0>	2.8367867469787598 seconds	 sig: 50
<function sig_v4 at 0x109772268>	2.682001829147339 seconds	 sig: 70
<function sig_v5 at 0x1097722f0>	2.618752956390381 seconds	 sig: 48

$ python spearman-perm.py 0.05 1000
<function sig_v0 at 0x1036792f0>	3.6979241371154785 seconds	 sig: 62
<function sig_v1 at 0x10bed9950>	4.586740016937256 seconds	 sig: 50
<function sig_v2 at 0x10bed9b70>	3.1884469985961914 seconds	 sig: 54
<function sig_v3 at 0x10c0291e0>	3.5503461360931396 seconds	 sig: 39
<function sig_v4 at 0x10c02e268>	3.4625608921051025 seconds	 sig: 53
<function sig_v5 at 0x10c02e2f0>	2.928835868835449 seconds	 sig: 61

$ python spearman-perm.py 0.05 10000
<function sig_v0 at 0x1041062f0>	31.09247589111328 seconds	 sig: 489
<function sig_v1 at 0x10ca32950>	31.629493951797485 seconds	 sig: 555
<function sig_v2 at 0x10ca32b70>	32.653910875320435 seconds	 sig: 539
<function sig_v3 at 0x10cb821e0>	35.457908153533936 seconds	 sig: 572
<function sig_v4 at 0x10cb87268>	30.291733980178833 seconds	 sig: 542
<function sig_v5 at 0x10cb872f0>	30.00481629371643 seconds	 sig: 559

spearman-perm.py

"""
Comparing runtime of a for loop vs comprehensions for a permutation test

Usage:
    python spearman-perm.py <p_value> <num_sims>
"""

import multiprocessing as mp
import numpy as np
import scipy.stats as stats
import sys
import time


def main():
    p_value = float(sys.argv[1])
    num_sims = int(sys.argv[2])

    col1List = [0.0167602, 0.02258471, 0.016119, 0.00932171, 0.01220057, 0.01689389, 0.28883007, 0.18259194, 0.5270876,0.0471272]
    col2List = [3.77333333e-01, 3.42500000e-01, 2.18166667e-01, -6.16666667e-03, 7.26666667e-02, -2.96666667e-02,1.65000000e-02, -1.25833333e-01, 9.66666667e-02, -1.51666667e-02]

    for method in (sig_v0, sig_v1, sig_v2, sig_v3, sig_v4, sig_v5):
        start = time.time()
        sig = method(col1List, col2List, num_sims, p_value)
        end = time.time()
        print(f"{method}\t{end - start} seconds\t sig: {sig}")


def sig_v0(col1List, col2List, num_sims, p_value):
    """
    Simple for loop method
    """
    results = list()
    for i in range(num_sims):
        result = stats.mstats.spearmanr(np.random.permutation(col1List), col2List)
        if result.pvalue <= p_value:
            results.append(1)
    return sum(results)

def sig_v1(col1List, col2List, num_sims, p_value):
    """
    Cathy's original comprehension method
    """
    return sum(1 for result in [stats.mstats.spearmanr(np.random.permutation(col1List), col2List) for i in range(num_sims)] if result.pvalue <= p_value)

def sig_v2(col1List, col2List, num_sims, p_value):
    """
    Kelly's 1st attempt to make the comprehension method faster
    """
    return sum(stats.mstats.spearmanr(np.random.permutation(col1List), col2List).pvalue <= p_value for i in range(num_sims))

def sig_v3(col1List, col2List, num_sims, p_value):
    """
    Kelly's 2nd attempt to make the comprehension method faster
    """
    return sum(1 for i in range(num_sims) if stats.mstats.spearmanr(np.random.permutation(col1List), col2List).pvalue <= p_value)

def sig_v4(col1List, col2List, num_sims, p_value):
    """
    Try using shuffle in a for loop instead of permutation
    """
    results = list()
    for i in range(num_sims):
        np.random.shuffle(col1List)
        result = stats.mstats.spearmanr(col1List, col2List)
        if result.pvalue <= p_value:
            results.append(1)
    return sum(results)

def sig_v5(col1List, col2List, num_sims, p_value):
    """
    Shuffle with a hackish comprehension
    """
    return sum(1 for i in range(num_sims) if (np.random.shuffle(col1List), stats.mstats.spearmanr(col1List, col2List))[1].pvalue <= p_value)


if __name__ == "__main__":
    main()