Python benchmarking environment setup script https://bugs.python.org/issue32616 https://github.com/Homebrew/homebrew-core/issues/22743

python-benchmark-setup-macos

#!/usr/bin/env zsh

# macOS 10.13.2.

setopt xtrace errexit

ROOTDIR=$PWD/python-benchmark

brew install python2 python3 llvm llvm@4 llvm@3.9

mkdir -p $ROOTDIR
cd $ROOTDIR
[[ -d cpython ]] || git clone https://github.com/python/cpython.git

[[ -f get-pip.py ]] || curl -fsSL -o get-pip.py https://bootstrap.pypa.io/get-pip.py

[[ -f test.py ]] || cat >test.py <<'EOF'
import time

def f(n):
    while n > 0:
        n -= 1

start = time.time()
f(50000000)
stop = time.time()
print('%.6f' % (stop - start))
EOF

[[ -f bench ]] || cat >bench <<'EOF'
#!/usr/bin/env zsh

pythons=(
    /usr/bin/python2.7
    /usr/local/bin/python2.7
    dist/python27-apple-clang-900/bin/python2.7
    dist/python27-clang-3.9/bin/python2.7
    dist/python27-clang-4.0/bin/python2.7
    dist/python27-clang-5.0/bin/python2.7
    /usr/local/bin/python3.6
    dist/python-master-apple-clang-900/bin/python3
    dist/python-master-clang-3.9/bin/python3
    dist/python-master-clang-4.0/bin/python3
    dist/python-master-clang-5.0/bin/python3
)

for python in $pythons; do
    printf '%s\t%s\n' "$($python test.py)" $python
done
EOF
chmod +x bench

cd cpython

# build <CC> <prefix>
build () {
    [[ -e $2 ]] && return

    trap 'git checkout .; rm -f setup.py.bak; make distclean' EXIT

    # Help set.py find Homebrew's OpenSSL; otherwise we end up with a python
    # without _ssl, and get-pip.py won't work.
    sed -i.bak 's:/usr/local/ssl:/usr/local/opt/openssl:' setup.py

    CC=$1 ./configure --prefix=$2
    make install -j8
}

# getpip <prefix>
#
# We also make sure pip is available so that further, more rigorous
# benchmarking (e.g. via the performance package) is possible.
getpip () {
    [[ -f $1/bin/pip ]] || $1/bin/python $ROOTDIR/get-pip.py
}

export CPPFLAGS='-I/usr/local/opt/openssl/include'
export LDFLAGS='-L/usr/local/opt/openssl/lib'
export PKG_CONFIG_PATH='/usr/local/opt/openssl/lib/pkgconfig'

git checkout 2.7
build /usr/bin/clang $ROOTDIR/dist/python27-apple-clang-900
build /usr/local/opt/llvm@3.9/bin/clang-3.9 $ROOTDIR/dist/python27-clang-3.9
getpip $ROOTDIR/dist/python27-clang-3.9
build /usr/local/opt/llvm@4/bin/clang-4.0 $ROOTDIR/dist/python27-clang-4.0
getpip $ROOTDIR/dist/python27-clang-4.0
build /usr/local/opt/llvm/bin/clang-5.0 $ROOTDIR/dist/python27-clang-5.0
getpip $ROOTDIR/dist/python27-clang-5.0

# I tried to build with gcc too, but gcc (from 4.x to latest 7.2) always choke
# on Security.framework:
#   /System/Library/Frameworks/Security.framework/Headers/Authorization.h:193:7: error: variably modified 'bytes' at file scope

git checkout master
build /usr/bin/clang $ROOTDIR/dist/python-master-apple-clang-900
build /usr/local/opt/llvm@3.9/bin/clang-3.9 $ROOTDIR/dist/python-master-clang-3.9
build /usr/local/opt/llvm@4/bin/clang-4.0 $ROOTDIR/dist/python-master-clang-4.0
build /usr/local/opt/llvm/bin/clang-5.0 $ROOTDIR/dist/python-master-clang-5.0

python-benchmark-setup-ubuntu

#!/usr/bin/env zsh

# Prerequisites (Artful):
# sudo apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev clang-3.9 clang-4.0 clang-5.0 gcc-5 gcc-6 gcc-7 python2.7 python3.6

setopt xtrace errexit

ROOTDIR=$PWD/python-benchmark

mkdir -p $ROOTDIR
cd $ROOTDIR
[[ -d cpython ]] || git clone https://github.com/python/cpython.git

[[ -f get-pip.py ]] || curl -fsSL -o get-pip.py https://bootstrap.pypa.io/get-pip.py

[[ -f test.py ]] || cat >test.py <<'EOF'
import time

def f(n):
    while n > 0:
        n -= 1

start = time.time()
f(50000000)
stop = time.time()
print('%.6f' % (stop - start))
EOF

[[ -f bench ]] || cat >bench <<'EOF'
#!/usr/bin/env zsh

pythons=(
    /usr/bin/python2.7
    dist/python27-clang-3.9/bin/python2.7
    dist/python27-clang-4.0/bin/python2.7
    dist/python27-clang-5.0/bin/python2.7
    dist/python27-gcc-5/bin/python2.7
    dist/python27-gcc-6/bin/python2.7
    dist/python27-gcc-7/bin/python2.7
    /usr/bin/python3.6
    dist/python-master-clang-3.9/bin/python3
    dist/python-master-clang-4.0/bin/python3
    dist/python-master-clang-5.0/bin/python3
    dist/python-master-gcc-5/bin/python3
    dist/python-master-gcc-6/bin/python3
    dist/python-master-gcc-7/bin/python3
)

for python in $pythons; do
    printf '%s\t%s\n' "$($python test.py)" $python
done
EOF
chmod +x bench

cd cpython

# build <CC> <prefix>
build () {
    [[ -e $2 ]] && return

    trap 'make distclean' EXIT

    CC=$1 ./configure --prefix=$2
    make install -j8
}

# getpip <prefix>
#
# We also make sure pip is available so that further, more rigorous
# benchmarking (e.g. via the performance package) is possible.
getpip () {
    [[ -f $1/bin/pip ]] || $1/bin/python $ROOTDIR/get-pip.py
}

git checkout 2.7
build /usr/bin/clang-3.9 $ROOTDIR/dist/python27-clang-3.9
getpip $ROOTDIR/dist/python27-clang-3.9
build /usr/bin/clang-4.0 $ROOTDIR/dist/python27-clang-4.0
getpip $ROOTDIR/dist/python27-clang-4.0
build /usr/bin/clang-5.0 $ROOTDIR/dist/python27-clang-5.0
getpip $ROOTDIR/dist/python27-clang-5.0
build /usr/bin/gcc-5 $ROOTDIR/dist/python27-gcc-5
getpip $ROOTDIR/dist/python27-gcc-5
build /usr/bin/gcc-6 $ROOTDIR/dist/python27-gcc-6
getpip $ROOTDIR/dist/python27-gcc-6
build /usr/bin/gcc-7 $ROOTDIR/dist/python27-gcc-7
getpip $ROOTDIR/dist/python27-gcc-7

git checkout master
build /usr/bin/clang-3.9 $ROOTDIR/dist/python-master-clang-3.9
build /usr/bin/clang-4.0 $ROOTDIR/dist/python-master-clang-4.0
build /usr/bin/clang-5.0 $ROOTDIR/dist/python-master-clang-5.0
build /usr/bin/gcc-5 $ROOTDIR/dist/python-master-gcc-5
build /usr/bin/gcc-6 $ROOTDIR/dist/python-master-gcc-6
build /usr/bin/gcc-7 $ROOTDIR/dist/python-master-gcc-7

macOS 10.13.2:

2.082144	/usr/bin/python2.7
7.964049	/usr/local/bin/python2.7
8.750652	dist/python27-apple-clang-900/bin/python2.7
8.476405	dist/python27-clang-3.9/bin/python2.7
8.625660	dist/python27-clang-4.0/bin/python2.7
1.760096	dist/python27-clang-5.0/bin/python2.7
3.254814	/usr/local/bin/python3.6
2.864716	dist/python-master-apple-clang-900/bin/python3
3.071757	dist/python-master-clang-3.9/bin/python3
2.925192	dist/python-master-clang-4.0/bin/python3
2.908782	dist/python-master-clang-5.0/bin/python3

Ubuntu Artful in VirtualBox:

1.475095	/usr/bin/python2.7
8.576817	dist/python27-clang-3.9/bin/python2.7
8.165588	dist/python27-clang-4.0/bin/python2.7
1.779193	dist/python27-clang-5.0/bin/python2.7
1.728321	dist/python27-gcc-5/bin/python2.7
1.570040	dist/python27-gcc-6/bin/python2.7
1.604617	dist/python27-gcc-7/bin/python2.7
2.323037	/usr/bin/python3.6
2.964338	dist/python-master-clang-3.9/bin/python3
3.054277	dist/python-master-clang-4.0/bin/python3
2.734908	dist/python-master-clang-5.0/bin/python3
2.490278	dist/python-master-gcc-5/bin/python3
2.494691	dist/python-master-gcc-6/bin/python3
2.642277	dist/python-master-gcc-7/bin/python3