JMH Clojure demo

bench.clj

;; scripts/bench.clj

(ns bench
  (:require [criterium.core :as crit]
            [jmh.core :as jmh]
            [clojure.java.io :as io]
            [clojure.edn :as edn]
            [demo.core :as demo]
            [demo.utils :as utils]
            [clojure.pprint :refer [pprint]]))

(def bench-env
  (-> "benchmarks.edn" io/resource slurp edn/read-string))

(def bench-counts [31 100000])

(def bench-fns [[:str utils/make-str]
                [:vec utils/make-vec]])

(def bench-opts
  {:type :quick
   :params {:count bench-counts}
   :profilers ["gc"]
   :progress false})

;;;

(defmulti run identity)

(defn -main [type & [opt]]
  (-> (if-not opt
        (run (read-string type))
        (run (map read-string [type opt])))
      doall pprint))

;;;

(defmethod run :forked [_]
  (->> (jmh/run bench-env bench-opts)
       (map #(select-keys % [:name :params :score]))))

(defmacro run-expr [runner make count opts]
  `(let [count# ~count
         idx# (* 0.5 count#)
         val# (~make count#)]
     (~runner (demo/value-at val# idx#) ~opts)))

(defmethod run :manual [_]
  (for [[name make] bench-fns
        count bench-counts
        :let [result (run-expr jmh/run-expr make count bench-opts)]]
    {:tag [name count], :score (:score result)}))

(defmethod run :criterium [_]
  (for [[name make] bench-fns
        count bench-counts
        :let [result (run-expr crit/quick-benchmark make count {})]]
    {:tag [name count], :mean (first (:mean result))}))

(defmethod run [:manual :interleaved] [_]
  (for [count bench-counts
        [name make] bench-fns
        :let [result (run-expr jmh/run-expr make count bench-opts)]]
    {:tag [name count], :score (:score result)}))

(defmethod run [:criterium :interleaved] [_]
  (for [count bench-counts
        [name make] bench-fns
        :let [result (run-expr crit/quick-benchmark make count {})]]
    {:tag [name count], :mean (first (:mean result))}))

benchmarks.edn

;; resources/benchmarks.edn

{:benchmarks
 [{:name :str, :fn demo.core/value-at, :args [:state/string, :state/index]}
  {:name :vec, :fn demo.core/value-at, :args [:state/vector, :state/index]}]

 :states
 {:index {:fn (partial * 0.5), :args [:param/count]}
  :string {:fn demo.utils/make-str, :args [:param/count]}
  :vector {:fn demo.utils/make-vec, :args [:param/count]}}

 :params {:count 10}}

core.clj

;; src/demo/core.clj

(ns demo.core)

(defprotocol ValueAt
  (value-at [x idx]))

(extend-protocol ValueAt
  clojure.lang.Indexed
  (value-at [i idx]
    (.nth i idx))
  CharSequence
  (value-at [s idx]
    (.charAt s idx)))

project.clj

(defproject demo "0.1.0-SNAPSHOT"
  :dependencies [[org.clojure/clojure "1.8.0"]
                 [criterium "0.4.4"]
                 [jmh-clojure "0.1.0"]]
  :jvm-opts []
  :profiles {:dev {:source-paths ["scripts"]}})

results.edn

;; Forked results (executions per second, higher is better)

({:name :str, :params {:count 31},     :score [1.3469278621324E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45137213786726E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [8.3640879205095E7 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.3892304482946E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.22414032556451E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45545380106672E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.4558060099421E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [6.8808657201576E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.07502303575012E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45459846564931E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.4552407028013E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.340469091291E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.44959801438209E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45485370497829E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45550038851249E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.5783753539823E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.45377356361152E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45306872589013E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45496569170135E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [6.8862867173066E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.45065354945518E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45225731925461E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45558510380066E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.3488823120479E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.45493157001018E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45389462773361E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45466722236274E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.3498598281824E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.45267965676815E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45535368555091E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45600300142722E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.4435645914128E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.45256052398033E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [1.45532820302949E8 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45515307263459E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [8.3537792698387E7 "ops/s"]})

({:name :str, :params {:count 31},     :score [1.44517404482434E8 "ops/s"]}
 {:name :str, :params {:count 100000}, :score [6.329898790254E7 "ops/s"]}
 {:name :vec, :params {:count 31},     :score [1.45323558096936E8 "ops/s"]}
 {:name :vec, :params {:count 100000}, :score [6.5879473106668E7 "ops/s"]})

;; Manual results

({:tag [:str 31],     :score [1.3379804237942E8 "ops/s"]}
 {:tag [:str 100000], :score [1.39460799485597E8 "ops/s"]}
 {:tag [:vec 31],     :score [7.9531529265816E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.8703015698519E7 "ops/s"]})

({:tag [:str 31],     :score [4.9937189944882E7 "ops/s"]}
 {:tag [:str 100000], :score [5.1584022177376E7 "ops/s"]}
 {:tag [:vec 31],     :score [7.5850795813495E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.640802413702E7 "ops/s"]})

({:tag [:str 31],     :score [5.3193140407946E7 "ops/s"]}
 {:tag [:str 100000], :score [5.1623346247677E7 "ops/s"]}
 {:tag [:vec 31],     :score [8.3658996296438E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.0548827596392E7 "ops/s"]})

({:tag [:str 31],     :score [1.21206678853589E8 "ops/s"]}
 {:tag [:str 100000], :score [1.28637620352247E8 "ops/s"]}
 {:tag [:vec 31],     :score [9.0435375554475E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.0859761443865E7 "ops/s"]})

({:tag [:str 31],     :score [5.2582382032095E7 "ops/s"]}
 {:tag [:str 100000], :score [5.1740742613911E7 "ops/s"]}
 {:tag [:vec 31],     :score [8.3588761914769E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.0574105811449E7 "ops/s"]})

({:tag [:str 31],     :score [1.33580908961255E8 "ops/s"]}
 {:tag [:str 100000], :score [1.33815496584507E8 "ops/s"]}
 {:tag [:vec 31],     :score [4.3557084244811E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.6936015069735E7 "ops/s"]})

({:tag [:str 31],     :score [1.29379504484452E8 "ops/s"]}
 {:tag [:str 100000], :score [1.39374070077195E8 "ops/s"]}
 {:tag [:vec 31],     :score [7.9548957250226E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.8700906284802E7 "ops/s"]})

({:tag [:str 31],     :score [1.29748211788299E8 "ops/s"]}
 {:tag [:str 100000], :score [1.28681488108041E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.5160988647899E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.1889208726533E7 "ops/s"]})

({:tag [:str 31],     :score [1.35232808326822E8 "ops/s"]}
 {:tag [:str 100000], :score [1.28666423276058E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.5877307104252E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.1686637961537E7 "ops/s"]})

({:tag [:str 31],     :score [1.28642546691396E8 "ops/s"]}
 {:tag [:str 100000], :score [1.19532166095737E8 "ops/s"]}
 {:tag [:vec 31],     :score [7.2408742569931E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.4831532522043E7 "ops/s"]})

;; Manual results (interleaved)

({:tag [:str 31],     :score [5.0169123770581E7 "ops/s"]}
 {:tag [:vec 31],     :score [8.3671183975389E7 "ops/s"]}
 {:tag [:str 100000], :score [8.367388431667E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.1732392822386E7 "ops/s"]})

({:tag [:str 31],     :score [1.28703495047955E8 "ops/s"]}
 {:tag [:vec 31],     :score [4.5341468637465E7 "ops/s"]}
 {:tag [:str 100000], :score [8.5430194915798E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.5844534262763E7 "ops/s"]})

({:tag [:str 31],     :score [5.2544966955773E7 "ops/s"]}
 {:tag [:vec 31],     :score [8.5674406192978E7 "ops/s"]}
 {:tag [:str 100000], :score [8.5241125675328E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.1359347479839E7 "ops/s"]})

({:tag [:str 31],     :score [1.28725659977987E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.4931939178099E7 "ops/s"]}
 {:tag [:str 100000], :score [8.4740128604998E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.1636447829743E7 "ops/s"]})

({:tag [:str 31],     :score [1.22996371614272E8 "ops/s"]}
 {:tag [:vec 31],     :score [4.3311513570353E7 "ops/s"]}
 {:tag [:str 100000], :score [8.7452576605875E7 "ops/s"]}
 {:tag [:vec 100000], :score [3.5849990949054E7 "ops/s"]})

({:tag [:str 31],     :score [1.34467890183543E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.5433016385565E7 "ops/s"]}
 {:tag [:str 100000], :score [8.1106489042152E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.8555422920146E7 "ops/s"]})

({:tag [:str 31],     :score [1.33957176603751E8 "ops/s"]}
 {:tag [:vec 31],     :score [7.9403522759931E7 "ops/s"]}
 {:tag [:str 100000], :score [8.1264641119648E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.5273030239129E7 "ops/s"]})

({:tag [:str 31],     :score [1.28778065559463E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.3298559161994E7 "ops/s"]}
 {:tag [:str 100000], :score [8.2693845631514E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.9460716770459E7 "ops/s"]})

({:tag [:str 31],     :score [1.15469910227268E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.563699695283E7 "ops/s"]}
 {:tag [:str 100000], :score [8.4208009188159E7 "ops/s"]}
 {:tag [:vec 100000], :score [6.1513675775968E7 "ops/s"]})

({:tag [:str 31],     :score [1.22974894169097E8 "ops/s"]}
 {:tag [:vec 31],     :score [8.3297076696679E7 "ops/s"]}
 {:tag [:str 100000], :score [8.1177634323827E7 "ops/s"]}
 {:tag [:vec 100000], :score [5.7993109879722E7 "ops/s"]})

;; Criterium results (execution time mean, lower is better)

({:tag [:str 31],     :mean 7.496499932982252E-9}
 {:tag [:str 100000], :mean 7.495521144471977E-9}
 {:tag [:vec 31],     :mean 2.073846695254863E-8}
 {:tag [:vec 100000], :mean 2.6258323467357757E-8})

({:tag [:str 31],     :mean 1.1512071151838241E-8}
 {:tag [:str 100000], :mean 1.1464008702949106E-8}
 {:tag [:vec 31],     :mean 1.2651455571620972E-8}
 {:tag [:vec 100000], :mean 1.72661759725987E-8})

({:tag [:str 31],     :mean 9.294673987795223E-9}
 {:tag [:str 100000], :mean 9.283602472545468E-9}
 {:tag [:vec 31],     :mean 2.019119469530449E-8}
 {:tag [:vec 100000], :mean 2.4699610766954523E-8})

({:tag [:str 31],     :mean 7.634069393422915E-9}
 {:tag [:str 100000], :mean 7.634177515158605E-9}
 {:tag [:vec 31],     :mean 1.121934610543802E-8}
 {:tag [:vec 100000], :mean 1.6711752063751384E-8})

({:tag [:str 31],     :mean 1.3601164521977289E-8}
 {:tag [:str 100000], :mean 1.3755707666353809E-8}
 {:tag [:vec 31],     :mean 1.524882618491252E-8}
 {:tag [:vec 100000], :mean 1.9745483331802685E-8})

({:tag [:str 31],     :mean 7.196863692456165E-9}
 {:tag [:str 100000], :mean 7.193011552627127E-9}
 {:tag [:vec 31],     :mean 2.163042395543514E-8}
 {:tag [:vec 100000], :mean 2.5995142125606275E-8})

({:tag [:str 31],     :mean 7.5013785514991E-9}
 {:tag [:str 100000], :mean 7.485281213334384E-9}
 {:tag [:vec 31],     :mean 1.3800493832083598E-8}
 {:tag [:vec 100000], :mean 1.829993009411832E-8})

({:tag [:str 31],     :mean 7.197700589281779E-9}
 {:tag [:str 100000], :mean 7.2143376963592905E-9}
 {:tag [:vec 31],     :mean 1.2533533780793485E-8}
 {:tag [:vec 100000], :mean 1.846818563125412E-8})

({:tag [:str 31],     :mean 1.3011510240160115E-8}
 {:tag [:str 100000], :mean 1.2972883541193703E-8}
 {:tag [:vec 31],     :mean 1.4534519717577697E-8}
 {:tag [:vec 100000], :mean 1.960916696048152E-8})

({:tag [:str 31],     :mean 1.2021981073108377E-8}
 {:tag [:str 100000], :mean 1.2118540566188468E-8}
 {:tag [:vec 31],     :mean 1.3527370957359275E-8}
 {:tag [:vec 100000], :mean 1.797358250639553E-8})

;; Criterium results (interleaved)

({:tag [:str 31],     :mean 1.1660557109605627E-8}
 {:tag [:vec 31],     :mean 1.2817969469973787E-8}
 {:tag [:str 100000], :mean 1.2950003988664044E-8}
 {:tag [:vec 100000], :mean 1.7265563901707005E-8})

({:tag [:str 31],     :mean 1.1735149541983066E-8}
 {:tag [:vec 31],     :mean 1.3182690185716178E-8}
 {:tag [:str 100000], :mean 1.3285753469395899E-8}
 {:tag [:vec 100000], :mean 1.7576996628799036E-8})

({:tag [:str 31],     :mean 7.1587079548787875E-9}
 {:tag [:vec 31],     :mean 1.2003327208866043E-8}
 {:tag [:str 100000], :mean 1.2675917930539426E-8}
 {:tag [:vec 100000], :mean 1.6611333923152853E-8})

({:tag [:str 31],     :mean 7.178347714127908E-9}
 {:tag [:vec 31],     :mean 2.074878101243958E-8}
 {:tag [:str 100000], :mean 1.1257477879167779E-8}
 {:tag [:vec 100000], :mean 2.507497219883394E-8})

({:tag [:str 31],     :mean 1.46302449898574E-8}
 {:tag [:vec 31],     :mean 1.445689384147269E-8}
 {:tag [:str 100000], :mean 1.5715904222136864E-8}
 {:tag [:vec 100000], :mean 2.0223160397713342E-8})

({:tag [:str 31],     :mean 7.202539270010169E-9}
 {:tag [:vec 31],     :mean 2.1399697861856655E-8}
 {:tag [:str 100000], :mean 1.1090423549118608E-8}
 {:tag [:vec 100000], :mean 2.6647673077790563E-8})

({:tag [:str 31],     :mean 7.509585937742151E-9}
 {:tag [:vec 31],     :mean 1.3277826996888652E-8}
 {:tag [:str 100000], :mean 1.3252078653035764E-8}
 {:tag [:vec 100000], :mean 1.7871560254375556E-8})

({:tag [:str 31],     :mean 7.49510089876381E-9}
 {:tag [:vec 31],     :mean 1.3147760745894432E-8}
 {:tag [:str 100000], :mean 1.3043634844251888E-8}
 {:tag [:vec 100000], :mean 1.7821677353883307E-8})

({:tag [:str 31],     :mean 7.172475664416443E-9}
 {:tag [:vec 31],     :mean 2.194415400094894E-8}
 {:tag [:str 100000], :mean 1.152966796042752E-8}
 {:tag [:vec 100000], :mean 2.6786212896148163E-8})

({:tag [:str 31],     :mean 6.81464854393918E-9}
 {:tag [:vec 31],     :mean 1.2774899860311153E-8}
 {:tag [:str 100000], :mean 1.321789260427435E-8}
 {:tag [:vec 100000], :mean 1.713625275340903E-8})

run.sh

java -version
# => openjdk version "1.8.0_131"
# => OpenJDK Runtime Environment (build 1.8.0_131-8u131-b11-2ubuntu1.17.04.3-b11)
# => OpenJDK 64-Bit Server VM (build 25.131-b11, mixed mode)

lein classpath | sed -r 's,.*/(clojure-[0-9.]*)\.jar($|:.*),\1,'
# => clojure-1.8.0

opts=(
    ':forked'
    ':manual'
    ':manual :interleaved'
    ':criterium'
    ':criterium :interleaved'
)

iterations=${1:-10}

for opts in "${opts[@]}"; do
    echo $opts
    for _ in $(seq "$iterations"); do
        lein run -m bench $opts
    done
done

utils.clj

;; src/demo/utils.clj

(ns demo.utils)

(defn make-str [n]
  (apply str (repeat n \x)))

(defn make-vec [n]
  (vec (make-str n)))