Strange results when benchmarking FFI bindings using criterion library in Haskell
module Main where | |
import Criterion.Main | |
import Data.Vector.Storable hiding (copy) | |
import Control.Monad (liftM) | |
import Foreign hiding (unsafePerformIO) | |
import Foreign.C | |
import System.IO.Unsafe (unsafePerformIO) | |
foreign import ccall unsafe "copy.h" | |
c_copy :: Ptr CDouble -> CInt -> IO (Ptr CDouble) | |
signal :: Vector Double | |
signal = fromList [1.0..16384.0] | |
copy :: Vector Double -> Vector Double | |
copy sig = unsafePerformIO $ do | |
let (fpSig, _, lenSig) = unsafeToForeignPtr sig | |
pLattice <- liftM castPtr $ withForeignPtr fpSig $ \ptrSig -> | |
c_copy (castPtr ptrSig) (fromIntegral lenSig) | |
fpLattice <- newForeignPtr finalizerFree pLattice | |
return $ unsafeFromForeignPtr0 fpLattice lenSig | |
main :: IO () | |
main = defaultMain [ | |
bgroup "FFI" [ | |
bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
, bench "C binding" $ whnf copy signal | |
] | |
] |
$ ghc -O2 -Wall -optc -std=c99 ffi_crit.hs copy.c | |
$ ./ffi_crit -g | |
warming up | |
estimating clock resolution... | |
mean is 1.431904 us (640001 iterations) | |
found 4750 outliers among 639999 samples (0.7%) | |
3581 (0.6%) high severe | |
estimating cost of a clock call... | |
mean is 45.00523 ns (10 iterations) | |
found 1 outliers among 10 samples (10.0%) | |
1 (10.0%) high severe | |
benchmarking FFI/C binding | |
mean: 17.44777 us, lb 16.82549 us, ub 19.84387 us, ci 0.950 | |
std dev: 5.627304 us, lb 968.1911 ns, ub 13.18222 us, ci 0.950 | |
found 5 outliers among 100 samples (5.0%) | |
5 (5.0%) high severe | |
variance introduced by outliers: 97.856% | |
variance is severely inflated by outliers | |
benchmarking FFI/C binding | |
mean: 45.46269 us, lb 45.17545 us, ub 46.01435 us, ci 0.950 | |
std dev: 1.950915 us, lb 1.169448 us, ub 3.201935 us, ci 0.950 | |
found 8 outliers among 100 samples (8.0%) | |
4 (4.0%) high mild | |
4 (4.0%) high severe | |
variance introduced by outliers: 40.503% | |
variance is moderately inflated by outliers | |
benchmarking FFI/C binding | |
mean: 45.79727 us, lb 45.55681 us, ub 46.26911 us, ci 0.950 | |
std dev: 1.669191 us, lb 1.029116 us, ub 3.098384 us, ci 0.950 | |
found 5 outliers among 100 samples (5.0%) | |
2 (2.0%) high mild | |
3 (3.0%) high severe | |
variance introduced by outliers: 32.640% | |
variance is moderately inflated by outliers | |
benchmarking FFI/C binding | |
mean: 46.71757 us, lb 46.32679 us, ub 47.29715 us, ci 0.950 | |
std dev: 2.393483 us, lb 1.821208 us, ub 3.336483 us, ci 0.950 | |
found 10 outliers among 100 samples (10.0%) | |
6 (6.0%) high mild | |
4 (4.0%) high severe | |
variance introduced by outliers: 49.452% | |
variance is moderately inflated by outliers | |
benchmarking FFI/C binding | |
mean: 46.39661 us, lb 45.99686 us, ub 47.02149 us, ci 0.950 | |
std dev: 2.526114 us, lb 1.830455 us, ub 3.693685 us, ci 0.950 | |
found 9 outliers among 100 samples (9.0%) | |
5 (5.0%) high mild | |
4 (4.0%) high severe | |
variance introduced by outliers: 52.463% | |
variance is severely inflated by outliers | |
benchmarking FFI/C binding | |
mean: 46.60653 us, lb 46.20746 us, ub 47.35528 us, ci 0.950 | |
std dev: 2.708197 us, lb 1.660032 us, ub 4.443404 us, ci 0.950 | |
found 8 outliers among 100 samples (8.0%) | |
4 (4.0%) high mild | |
4 (4.0%) high severe | |
variance introduced by outliers: 55.497% | |
variance is severely inflated by outliers | |
benchmarking FFI/C binding | |
mean: 47.10585 us, lb 46.75674 us, ub 47.87401 us, ci 0.950 | |
std dev: 2.536477 us, lb 1.191372 us, ub 4.499381 us, ci 0.950 | |
found 5 outliers among 100 samples (5.0%) | |
2 (2.0%) high mild | |
3 (3.0%) high severe | |
variance introduced by outliers: 51.486% | |
variance is severely inflated by outliers | |
benchmarking FFI/C binding | |
mean: 46.13946 us, lb 45.84257 us, ub 46.66114 us, ci 0.950 | |
std dev: 1.955241 us, lb 1.280196 us, ub 3.142932 us, ci 0.950 | |
found 5 outliers among 100 samples (5.0%) | |
4 (4.0%) high severe | |
variance introduced by outliers: 39.537% | |
variance is moderately inflated by outliers | |
benchmarking FFI/C binding | |
mean: 46.33848 us, lb 46.08088 us, ub 46.83814 us, ci 0.950 | |
std dev: 1.784367 us, lb 1.018966 us, ub 2.972920 us, ci 0.950 | |
found 6 outliers among 100 samples (6.0%) | |
3 (3.0%) high mild | |
3 (3.0%) high severe | |
variance introduced by outliers: 35.557% | |
variance is moderately inflated by outliers |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
Notice that first run of a benchmark takes about 2,5 times shorter than others. This happens most of the times, though sometimes all benchmarks take about 45 us. Where does this extra time come from?