tfausak/brittany-benchmark.hs

## brittany-benchmark.hs
#! /usr/bin/env sh
set -o errexit

# https://github.com/haskell-servant/servant/issues/986

###############################################################################
# Print out various system information.

date
lsb_release --all
uname --all
stack --version

###############################################################################
# Generate an empty Stack project to install dependencies.

cat <<'END' > stack.yaml
resolver: lts-18.28
extra-deps:
    - servant-0.19
    - servant-server-0.19
END

cat <<'END' > example.cabal
build-type: Simple
cabal-version: >= 1.2
name: example
version: 0
library
    build-depends: base, servant, servant-server, wai, warp
    build-tool-depends: bench:bench
    exposed-modules: Example
END

cat <<'END' > Example.hs
module Example where
END

###############################################################################
# Install dependencies.

stack build

###############################################################################
# Generate source files.

mkdir --parents old new
for count in 1 2 4 8 16 32 64
do
    old=old/$count.hs
    cat <<'END' > $old
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE TypeOperators #-}

module M ( main ) where

import qualified Network.Wai as Wai
import qualified Network.Wai.Handler.Warp as Warp
import qualified Servant

main :: IO ()
main = Warp.run 3000 application

application :: Wai.Application
application = Servant.serve proxy server

proxy :: Servant.Proxy Api
proxy = Servant.Proxy

server :: Servant.Server Api
server
END

    {
        for n in $( seq 1 "$count" )
        do
            op=$( test "$n" = 1 && echo '=' || echo 'Servant.:<|>' )
            echo "    $op pure $n"
        done

        echo
        echo 'type Api'
        for n in $( seq 1 "$count" )
        do
            op=$( test "$n" = 1 && echo '=' || echo 'Servant.:<|>' )
            echo "    $op \"$n\" Servant.:> Servant.Get '[Servant.JSON] Int"
        done
    } >> "$old"

    new="new/$count.hs"
    cat <<'END' > "$new"
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE TypeOperators #-}

module M ( main ) where

import qualified Network.Wai as Wai
import qualified Network.Wai.Handler.Warp as Warp
import qualified Servant
import qualified Servant.API.Generic as Servant
import qualified Servant.Server.Generic as Servant
import qualified GHC.Generics as Generics

main :: IO ()
main = Warp.run 3000 application

application :: Wai.Application
application = Servant.genericServe routes

routes :: Routes Servant.AsServer
routes = Routes
END

    {
        for n in $( seq 1 "$count" )
        do
            op=$( test "$n" = 1 && echo '{' || echo ',' )
            echo "    $op _$n = pure $n"
        done
        echo '    }'

        echo
        echo 'data Routes routes = Routes'
        for n in $( seq 1 "$count" )
        do
            op=$( test "$n" = 1 && echo '{' || echo ',' )
            echo "    $op _$n :: routes Servant.:- \"$n\" Servant.:> Servant.Get '[Servant.JSON] Int"
        done
        echo '    } deriving (Generics.Generic)'
    } >> "$new"
done

###############################################################################
# Run benchmark.

ghc='ghc -fforce-recomp -O1'
exec stack exec -- bench \
    "$ghc old/1.hs" "$ghc new/1.hs" \
    "$ghc old/2.hs" "$ghc new/2.hs" \
    "$ghc old/4.hs" "$ghc new/4.hs" \
    "$ghc old/8.hs" "$ghc new/8.hs" \
    "$ghc old/16.hs" "$ghc new/16.hs" \
    "$ghc old/32.hs" "$ghc new/32.hs" \
    "$ghc old/64.hs" "$ghc new/64.hs"

## output.txt
Mon 14 Mar 2022 04:34:32 PM EDT

No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 20.04.4 LTS
Release:        20.04
Codename:       focal

Linux carton 5.4.0-100-generic #113-Ubuntu SMP Thu Feb 3 18:43:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux

Version 2.7.3, Git revision 7927a3aec32e2b2e5e4fb5be76d0d50eddcc197f x86_64 hpack-0.34.4

benchmarking bench/ghc -fforce-recomp -O1 old/1.hs
time                 530.1 ms   (NaN s .. 663.8 ms)
                     0.993 R²   (0.985 R² .. 1.000 R²)
mean                 555.7 ms   (535.0 ms .. 587.6 ms)
std dev              30.07 ms   (1.413 ms .. 37.60 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/1.hs
time                 848.3 ms   (774.5 ms .. 899.3 ms)
                     0.999 R²   (0.997 R² .. 1.000 R²)
mean                 859.7 ms   (847.8 ms .. 873.5 ms)
std dev              14.39 ms   (6.441 ms .. 20.10 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 old/2.hs
time                 605.6 ms   (592.9 ms .. 616.4 ms)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 605.4 ms   (603.6 ms .. 607.2 ms)
std dev              2.236 ms   (1.050 ms .. 2.914 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/2.hs
time                 936.3 ms   (918.0 ms .. 961.8 ms)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 942.0 ms   (936.1 ms .. 947.6 ms)
std dev              6.313 ms   (3.709 ms .. 8.893 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 old/4.hs
time                 770.6 ms   (739.6 ms .. 798.4 ms)
                     1.000 R²   (0.999 R² .. 1.000 R²)
mean                 784.9 ms   (774.9 ms .. 792.1 ms)
std dev              10.34 ms   (5.306 ms .. 13.27 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/4.hs
time                 1.194 s    (1.153 s .. 1.221 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 1.180 s    (1.170 s .. 1.185 s)
std dev              9.456 ms   (2.890 ms .. 12.46 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 old/8.hs
time                 903.1 ms   (854.1 ms .. 942.4 ms)
                     1.000 R²   (0.999 R² .. 1.000 R²)
mean                 915.0 ms   (905.4 ms .. 922.0 ms)
std dev              10.04 ms   (3.000 ms .. 12.57 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/8.hs
time                 1.988 s    (1.948 s .. 2.056 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 1.965 s    (1.957 s .. 1.978 s)
std dev              12.49 ms   (1.971 ms .. 16.18 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 old/16.hs
time                 1.243 s    (1.209 s .. 1.304 s)
                     1.000 R²   (0.999 R² .. NaN R²)
mean                 1.197 s    (1.173 s .. 1.213 s)
std dev              24.78 ms   (12.94 ms .. 34.90 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/16.hs
time                 3.397 s    (3.388 s .. 3.409 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 3.424 s    (3.412 s .. 3.437 s)
std dev              15.03 ms   (5.194 ms .. 19.96 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 old/32.hs
time                 1.921 s    (1.891 s .. 1.982 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 1.895 s    (1.887 s .. 1.908 s)
std dev              13.03 ms   (156.9 μs .. 17.33 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/32.hs
time                 7.270 s    (7.181 s .. 7.425 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 7.267 s    (7.241 s .. 7.282 s)
std dev              24.34 ms   (10.34 ms .. 31.21 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 old/64.hs
time                 4.480 s    (4.371 s .. 4.604 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 4.440 s    (4.418 s .. 4.463 s)
std dev              28.07 ms   (13.94 ms .. 36.28 ms)
variance introduced by outliers: 19% (moderately inflated)

benchmarking bench/ghc -fforce-recomp -O1 new/64.hs
time                 16.24 s    (16.19 s .. 16.27 s)
                     1.000 R²   (1.000 R² .. 1.000 R²)
mean                 16.23 s    (16.21 s .. 16.24 s)
std dev              14.71 ms   (6.151 ms .. 18.82 ms)
variance introduced by outliers: 19% (moderately inflated)
	#! /usr/bin/env sh
	set -o errexit

	# https://github.com/haskell-servant/servant/issues/986

	###############################################################################
	# Print out various system information.

	date
	lsb_release --all
	uname --all
	stack --version

	###############################################################################
	# Generate an empty Stack project to install dependencies.

	cat <<'END' > stack.yaml
	resolver: lts-18.28
	extra-deps:
	- servant-0.19
	- servant-server-0.19
	END

	cat <<'END' > example.cabal
	build-type: Simple
	cabal-version: >= 1.2
	name: example
	version: 0
	library
	build-depends: base, servant, servant-server, wai, warp
	build-tool-depends: bench:bench
	exposed-modules: Example
	END

	cat <<'END' > Example.hs
	module Example where
	END

	###############################################################################
	# Install dependencies.

	stack build

	###############################################################################
	# Generate source files.

	mkdir --parents old new
	for count in 1 2 4 8 16 32 64
	do
	old=old/$count.hs
	cat <<'END' > $old
	{-# LANGUAGE DataKinds #-}
	{-# LANGUAGE TypeOperators #-}

	module M ( main ) where

	import qualified Network.Wai as Wai
	import qualified Network.Wai.Handler.Warp as Warp
	import qualified Servant

	main :: IO ()
	main = Warp.run 3000 application

	application :: Wai.Application
	application = Servant.serve proxy server

	proxy :: Servant.Proxy Api
	proxy = Servant.Proxy

	server :: Servant.Server Api
	server
	END

	{
	for n in $( seq 1 "$count" )
	do
	op=$( test "$n" = 1 && echo '=' \|\| echo 'Servant.:<\|>' )
	echo " $op pure $n"
	done

	echo
	echo 'type Api'
	for n in $( seq 1 "$count" )
	do
	op=$( test "$n" = 1 && echo '=' \|\| echo 'Servant.:<\|>' )
	echo " $op \"$n\" Servant.:> Servant.Get '[Servant.JSON] Int"
	done
	} >> "$old"

	new="new/$count.hs"
	cat <<'END' > "$new"
	{-# LANGUAGE DataKinds #-}
	{-# LANGUAGE DeriveGeneric #-}
	{-# LANGUAGE TypeOperators #-}

	module M ( main ) where

	import qualified Network.Wai as Wai
	import qualified Network.Wai.Handler.Warp as Warp
	import qualified Servant
	import qualified Servant.API.Generic as Servant
	import qualified Servant.Server.Generic as Servant
	import qualified GHC.Generics as Generics

	main :: IO ()
	main = Warp.run 3000 application

	application :: Wai.Application
	application = Servant.genericServe routes

	routes :: Routes Servant.AsServer
	routes = Routes
	END

	{
	for n in $( seq 1 "$count" )
	do
	op=$( test "$n" = 1 && echo '{' \|\| echo ',' )
	echo " $op _$n = pure $n"
	done
	echo ' }'

	echo
	echo 'data Routes routes = Routes'
	for n in $( seq 1 "$count" )
	do
	op=$( test "$n" = 1 && echo '{' \|\| echo ',' )
	echo " $op _$n :: routes Servant.:- \"$n\" Servant.:> Servant.Get '[Servant.JSON] Int"
	done
	echo ' } deriving (Generics.Generic)'
	} >> "$new"
	done

	###############################################################################
	# Run benchmark.

	ghc='ghc -fforce-recomp -O1'
	exec stack exec -- bench \
	"$ghc old/1.hs" "$ghc new/1.hs" \
	"$ghc old/2.hs" "$ghc new/2.hs" \
	"$ghc old/4.hs" "$ghc new/4.hs" \
	"$ghc old/8.hs" "$ghc new/8.hs" \
	"$ghc old/16.hs" "$ghc new/16.hs" \
	"$ghc old/32.hs" "$ghc new/32.hs" \
	"$ghc old/64.hs" "$ghc new/64.hs"
	Mon 14 Mar 2022 04:34:32 PM EDT

	No LSB modules are available.
	Distributor ID: Ubuntu
	Description: Ubuntu 20.04.4 LTS
	Release: 20.04
	Codename: focal

	Linux carton 5.4.0-100-generic #113-Ubuntu SMP Thu Feb 3 18:43:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux

	Version 2.7.3, Git revision 7927a3aec32e2b2e5e4fb5be76d0d50eddcc197f x86_64 hpack-0.34.4

	benchmarking bench/ghc -fforce-recomp -O1 old/1.hs
	time 530.1 ms (NaN s .. 663.8 ms)
	0.993 R² (0.985 R² .. 1.000 R²)
	mean 555.7 ms (535.0 ms .. 587.6 ms)
	std dev 30.07 ms (1.413 ms .. 37.60 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/1.hs
	time 848.3 ms (774.5 ms .. 899.3 ms)
	0.999 R² (0.997 R² .. 1.000 R²)
	mean 859.7 ms (847.8 ms .. 873.5 ms)
	std dev 14.39 ms (6.441 ms .. 20.10 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 old/2.hs
	time 605.6 ms (592.9 ms .. 616.4 ms)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 605.4 ms (603.6 ms .. 607.2 ms)
	std dev 2.236 ms (1.050 ms .. 2.914 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/2.hs
	time 936.3 ms (918.0 ms .. 961.8 ms)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 942.0 ms (936.1 ms .. 947.6 ms)
	std dev 6.313 ms (3.709 ms .. 8.893 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 old/4.hs
	time 770.6 ms (739.6 ms .. 798.4 ms)
	1.000 R² (0.999 R² .. 1.000 R²)
	mean 784.9 ms (774.9 ms .. 792.1 ms)
	std dev 10.34 ms (5.306 ms .. 13.27 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/4.hs
	time 1.194 s (1.153 s .. 1.221 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 1.180 s (1.170 s .. 1.185 s)
	std dev 9.456 ms (2.890 ms .. 12.46 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 old/8.hs
	time 903.1 ms (854.1 ms .. 942.4 ms)
	1.000 R² (0.999 R² .. 1.000 R²)
	mean 915.0 ms (905.4 ms .. 922.0 ms)
	std dev 10.04 ms (3.000 ms .. 12.57 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/8.hs
	time 1.988 s (1.948 s .. 2.056 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 1.965 s (1.957 s .. 1.978 s)
	std dev 12.49 ms (1.971 ms .. 16.18 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 old/16.hs
	time 1.243 s (1.209 s .. 1.304 s)
	1.000 R² (0.999 R² .. NaN R²)
	mean 1.197 s (1.173 s .. 1.213 s)
	std dev 24.78 ms (12.94 ms .. 34.90 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/16.hs
	time 3.397 s (3.388 s .. 3.409 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 3.424 s (3.412 s .. 3.437 s)
	std dev 15.03 ms (5.194 ms .. 19.96 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 old/32.hs
	time 1.921 s (1.891 s .. 1.982 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 1.895 s (1.887 s .. 1.908 s)
	std dev 13.03 ms (156.9 μs .. 17.33 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/32.hs
	time 7.270 s (7.181 s .. 7.425 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 7.267 s (7.241 s .. 7.282 s)
	std dev 24.34 ms (10.34 ms .. 31.21 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 old/64.hs
	time 4.480 s (4.371 s .. 4.604 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 4.440 s (4.418 s .. 4.463 s)
	std dev 28.07 ms (13.94 ms .. 36.28 ms)
	variance introduced by outliers: 19% (moderately inflated)

	benchmarking bench/ghc -fforce-recomp -O1 new/64.hs
	time 16.24 s (16.19 s .. 16.27 s)
	1.000 R² (1.000 R² .. 1.000 R²)
	mean 16.23 s (16.21 s .. 16.24 s)
	std dev 14.71 ms (6.151 ms .. 18.82 ms)
	variance introduced by outliers: 19% (moderately inflated)