Skip to content

Instantly share code, notes, and snippets.

@wilkerlucio
Last active October 21, 2022 21:37
Show Gist options
  • Save wilkerlucio/f2a52d032101a5c436045e621e331b4c to your computer and use it in GitHub Desktop.
Save wilkerlucio/f2a52d032101a5c436045e621e331b4c to your computer and use it in GitHub Desktop.
{:deps {com.wsscode/async {:mvn/version "1.0.11"}
com.wsscode/pathom {:mvn/version "2.4.0"}
com.wsscode/pathom3 {:mvn/version "2022.03.04-alpha"}
criterium/criterium {:mvn/version "0.4.6"}
cheshire/cheshire {:mvn/version "5.10.1"}
org.clojure/clojure {:mvn/version "1.11.0-rc1"}}}
(ns com.wsscode.blog.benchmark
(:require
[cheshire.core :as json]
[clojure.core.async]
[clojure.set :as set]
[com.wsscode.async.async-clj :as async]
[com.wsscode.misc.coll :as coll]
[com.wsscode.misc.time :as time]
[com.wsscode.pathom.connect :as pc]
[com.wsscode.pathom.core :as p]
[com.wsscode.pathom3.connect.built-in.resolvers :as pbir]
[com.wsscode.pathom3.connect.indexes :as pci]
[com.wsscode.pathom3.connect.operation :as pco]
[com.wsscode.pathom3.connect.runner.async :as pcra]
[com.wsscode.pathom3.interface.async.eql :as p.a.eql]
[com.wsscode.pathom3.interface.eql :as p.eql]
[com.wsscode.promesa.bridges.core-async]
[criterium.core :as c]
[promesa.core :as pp]))
(defn format-time [mean]
(let [[factor unit] (c/scale-time mean)]
(c/format-value mean factor unit)))
(defn bench
"Runs `f` for `times` times, returns the average time to run it.
Example:
(bench 10000 #(do-something))"
[{:keys [name runner] :as b} entity tx]
(println name)
(let [r (runner entity tx)
results (c/quick-benchmark (r) {})
mean (-> results :sample-mean first)
mean-ms (* (double mean) 1000.0)]
(println (str mean-ms "ms"))
(-> b
(assoc
:sample-mean
mean-ms
:formatted-mean
(format-time mean))
(dissoc :runner))))
(defn bench-compare
"Compute multiple benchmarks together with names. Example:
(bench-compare 10000
{:op-a #(do-something)
:other-op #(do-something-else)})"
[runners entity tx]
(let [data (into [] (map #(bench % entity tx)) runners)
{:keys [sample-mean]} (apply min-key :sample-mean data)]
(mapv #(assoc % :variance (/ (:sample-mean %) sample-mean)) data)))
(defn bench->chart [runners entity tx]
(->> (bench-compare runners entity tx)
(into {}
(map (juxt :id :sample-mean)))))
; region env setup
(defn slow-param-transform [config]
(let [k (if (contains? config ::pco/resolve)
::pco/resolve
::pc/resolve)]
(update config k
(fn [resolve]
(fn [env input]
(when-let [delay (-> (or (-> env :ast :params)
(pco/params env))
:delay)]
(time/sleep-ms delay))
(resolve env input))))))
(defn items-resolver [count]
(let [kw (keyword (str "items-" count))]
(pco/resolver (symbol (str "items-" count))
{::pco/output [{kw [:id]}]
::pco/cache? false}
(fn [{:keys [n]} _]
{kw (mapv (fn [_] (hash-map :id (vswap! n inc))) (range count))}))))
(defn gen-resolvers [attr]
(let [attr-s (name attr)]
[(pco/resolver (symbol attr-s)
{::pco/input [:id]
::pco/output [attr]
::pco/transform slow-param-transform}
(fn [_ {:keys [id]}] {attr (* 10 id)}))
(let [batch-attr (keyword (str attr-s "-batch"))]
(pco/resolver (symbol (str attr-s "-batch"))
{::pco/input [:id]
::pco/output [batch-attr]
::pco/batch? true
::pco/transform slow-param-transform}
(fn [_ ids]
(mapv #(hash-map batch-attr (* 10 (:id %))) ids))))]))
(pco/defresolver customer-account [{:customer/keys [id]}]
{:account/id id})
(pco/defresolver customer-info [{:customer/keys [id]}]
{:customer/name "Name"
:customer/birthday "Bday"
:customer/genre "Genre"})
(pco/defresolver purchases [{:customer/keys [id]}]
{::pco/output
[{:customer/purchases
[:purchase/id
:purchase/amount
:purchase/title]}]}
{:customer/purchases
(mapv
#(array-map
:purchase/id %
:purchase/vendor (str "V" %)
:purchase/amount (rand-int 100))
(range 50))})
(pco/defresolver purchases-total [{:keys [customer/purchases]}]
{::pco/input
[{:customer/purchases
[:purchase/amount
:purchase/category]}]}
{:customer/purchases-total (transduce (map :purchase/amount) + purchases)})
(pco/defresolver chargebacks [{:keys [customer/id]}]
{:customer/chargebacks
[{:chargeback/date "date"
:chargeback/status "sts"}
{:chargeback/date "date"
:chargeback/status "sts"}
{:chargeback/date "date"
:chargeback/status "sts"}
{:chargeback/date "date"
:chargeback/status "sts"}
{:chargeback/date "date"
:chargeback/status "sts"}
{:chargeback/date "date"
:chargeback/status "sts"}]})
(pco/defresolver boletos [{:keys [customer/id]}]
{:customer/boletos
[{:boleto/id 1
:boleto/number "date"
:boleto/paid? "sts"}
{:boleto/id 2
:boleto/number "date"
:boleto/paid? "sts"}
{:boleto/id 3
:boleto/number "date"
:boleto/paid? "sts"}]})
(pco/defresolver credit-cards [{:keys [account/id]}]
{:account/credit-cards
[{:credit-card/id 1
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}
{:credit-card/id 2
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}
{:credit-card/id 3
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}
{:credit-card/id 4
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}
{:credit-card/id 5
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}
{:credit-card/id 6
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}
{:credit-card/id 7
:credit-card/masked-number "0321xxxxxxxxxxx3214"
:credit-card/status :credit-card.status/active}]})
(pco/defresolver purchase-category [inputs]
{::pco/input
[:purchase/id]
::pco/output
[:purchase/category]
::pco/batch? true}
(mapv #(assoc % :purchase/category (str "cat-" (rand-int 10))) inputs))
(pco/defresolver account-details [{:account/keys [id]}]
{:account/balance "balance"
:account/balance-future "future"
:account/onboarding-date "date"
:account/status "status"})
(pco/defresolver account-status-history [{:account/keys [id]}]
{:account/status-history
[{:status "on"
:at "date"}
{:status "off"
:at "date"}
{:status "on"
:at "date"}
{:status "off"
:at "date"}
{:status "on"
:at "date"}]})
(pco/defresolver customer-referrals [{:keys [customer/id]}]
{:customer/referrals
[{:customer/id 2}
{:customer/id 3}
{:customer/id 4}
{:customer/id 5}]})
(pco/defresolver customer-phones [{:keys [customer/id]}]
{:customer/phones
[{:phone/id 2
:phone/number "2312424141"}
{:phone/id 3
:phone/number "531541241"}]})
(defn delay-resolver [resolver delay-ms]
(pco/wrap-resolve resolver
(fn [resolve]
(fn [env input]
(if (::pcra/async-runner? env)
(if (::p/path env)
(async/go-promise
(async/<! (clojure.core.async/timeout delay-ms))
(async/<?maybe (resolve env input)))
(pp/do!
(pp/delay delay-ms)
(resolve env input)))
(do
(Thread/sleep delay-ms)
(resolve env input)))))))
(def registry
[(items-resolver 1)
(items-resolver 10)
(items-resolver 100)
(items-resolver 1000)
(items-resolver 10000)
(gen-resolvers :x)
(gen-resolvers :y)
(delay-resolver customer-account 50)
(delay-resolver customer-info 50)
(delay-resolver purchases 50)
(delay-resolver purchase-category 50)
(delay-resolver account-details 50)
(delay-resolver account-status-history 50)
(delay-resolver chargebacks 50)
(delay-resolver credit-cards 50)
(delay-resolver boletos 50)
(delay-resolver customer-referrals 50)
(delay-resolver customer-phones 50)
purchases-total
; complex route
(pbir/constantly-resolver :a 1)
(pbir/single-attr-resolver :g :b inc)
(pbir/constantly-resolver :c 2)
(pbir/constantly-resolver :e 1)
(update (pbir/constantly-resolver :e 3) :config assoc ::pco/op-name 'e1)
(pbir/single-attr-resolver :e :f inc)
(update (pbir/constantly-resolver :g 4) :config assoc ::pco/input [:c :f])
(update (pbir/constantly-resolver :h 5) :config assoc ::pco/input [:a :b])])
(def env-indexes
(pci/register registry))
(defn base-env [plan-cache]
(-> env-indexes
(assoc :n (volatile! 0))
(cond->
plan-cache
(assoc :com.wsscode.pathom3.connect.planner/plan-cache* plan-cache))
((requiring-resolve 'com.wsscode.pathom.viz.ws-connector.pathom3/connect-env)
"debug")))
(defn resolver-p3->p2 [resolver]
(let [{::pc/keys [transform] :as config} (pco/operation-config resolver)]
(-> config
(set/rename-keys {::pco/op-name ::pc/sym
::pco/input ::pc/input
::pco/output ::pc/output
::pco/batch? ::pc/batch?
::pco/cache? ::pc/cache})
(assoc ::pc/resolve (:resolve resolver))
(coll/update-if ::pc/input set)
(cond-> transform transform))))
(def p2-registry
(mapv resolver-p3->p2 (flatten registry)))
; endregion
; region runners
(defn run-p2-serial [entity query]
(let [parser
(p/parser
{::p/env {::p/reader [p/map-reader
pc/reader2
pc/open-ident-reader
p/env-placeholder-reader]
::p/placeholder-prefixes #{">"}}
::p/mutate pc/mutate
::p/plugins [(pc/connect-plugin {::pc/register p2-registry})
p/error-handler-plugin]})]
#(parser {:n (volatile! 0)
::p/entity (atom entity)}
query)))
(defn run-p2-async [entity query]
(let [parser
(p/async-parser
{::p/env {::p/reader [p/map-reader
pc/async-reader2
pc/open-ident-reader
p/env-placeholder-reader]
::pcra/async-runner? true
::p/placeholder-prefixes #{">"}}
::p/mutate pc/mutate-async
::p/plugins [(pc/connect-plugin {::pc/register p2-registry})
p/error-handler-plugin]})]
#(-> (parser {:n (volatile! 0)
::p/entity (atom entity)}
query)
promesa.core/promise
(deref))))
(defn run-p2-parallel [entity query]
(let [parser
(p/parallel-parser
{::p/env {::p/reader [p/map-reader
pc/parallel-reader
pc/open-ident-reader
p/env-placeholder-reader]
::pcra/async-runner? true
::p/placeholder-prefixes #{">"}}
::p/mutate pc/mutate-async
::p/plugins [(pc/connect-plugin {::pc/register p2-registry})
p/error-handler-plugin]})]
#(-> (parser {:n (volatile! 0)
::p/entity (atom entity)}
query)
promesa.core/promise
(deref))))
(defn run-p3-serial [entity query]
#(p.eql/process (base-env nil) entity query))
(defn run-p3-async [entity query]
#(deref (p.a.eql/process (base-env nil) entity query)))
(defn run-p3-parallel [entity query]
#(deref (p.a.eql/process (assoc (base-env nil) ::p.a.eql/parallel? true) entity query)))
(defn run-p3-serial-cached [entity query]
(let [cache (atom {})
env (base-env cache)]
(p.eql/process env entity query)
#(p.eql/process env entity query)))
(defn run-p3-async-cached [entity query]
(let [cache (atom {})
env (base-env cache)]
(deref (p.a.eql/process env entity query))
#(deref (p.a.eql/process env entity query))))
(defn run-p3-parallel-cached [entity query]
(let [cache (atom {})
env (assoc (base-env cache) ::p.a.eql/parallel? true)]
(deref (p.a.eql/process env entity query))
#(deref (p.a.eql/process env entity query))))
(def all-runners
[{:name "Pathom 3 Serial Cached Plan"
:id "p3-serial-cp"
:runner run-p3-serial-cached}
{:name "Pathom 3 Serial"
:id "p3-serial"
:runner run-p3-serial}
{:name "Pathom 3 Async Cached Plan"
:id "p3-async-cp"
:runner run-p3-async-cached}
{:name "Pathom 3 Async"
:id "p3-async"
:runner run-p3-async}
{:name "Pathom 3 Parallel Cached Plan"
:id "p3-parallel-cp"
:runner run-p3-parallel-cached}
{:name "Pathom 3 Parallel"
:id "p3-parallel"
:runner run-p3-parallel}
{:name "Pathom 2 Serial"
:id "p2-serial"
:runner run-p2-serial}
{:name "Pathom 2 Async"
:id "p2-async"
:runner run-p2-async}
{:name "Pathom 2 Parallel"
:id "p2-parallel"
:runner run-p2-parallel}])
; endregion
(def benchmarks
[{:name "Read a single attribute"
:entity {:x 1}
:eql [:x]}
{:name "Complex plan"
:entity {:x 1}
:eql [:h]}
{:name "1000 items sequence"
:entity {}
:eql [{:items-1000 [:x]}]}
{:name "1000 items in batch"
:entity {}
:eql [{:items-1000 [:x-batch]}]}
{:name "10 items with 10 children with 10 children"
:entity {}
:eql [{:items-10
[:x
{:items-10
[:y
{:items-10
[:x]}]}]}]}
{:name "100 items with 10 children, batch head"
:entity {}
:eql [{:items-100
[:x-batch
{:items-10
[:y]}]}]}
{:name "100 items with 10 children, batch tail"
:entity {}
:eql [{:items-100
[:y
{:items-10
[:x-batch]}]}]}
{:name "100 items, delay 200 batch"
:entity {}
:eql '[{:items-100
[(:x-batch {:delay 200})]}]}
{:name "10 items with 10 children, delay 20, batch"
:entity {}
:eql '[{:items-10
[{:items-10
[(:x-batch {:delay 20})]}]}]}
{:name "Many widgets"
:entity {:customer/id 1}
:eql [{:>/profile-info
[:customer/name
:customer/birthday
:customer/genre
:account/status]}
{:>/purchases
[{:customer/purchases
[:purchase/vendor
:purchase/amount
:purchase/category]}]}
{:>/account-info
[:account/status
:account/status-history
:account/onboarding-date
:account/balance
:account/balance-future]}
{:>/chargebacks
[{:customer/chargebacks
[:chargeback/date
:chargeback/status]}]}
{:>/credit-cards
[{:account/credit-cards
[:credit-card/masked-number
:credit-card/status]}]}
{:>/boletos
[{:customer/boletos
[:boleto/number
:boleto/paid?]}]}
{:>/mgm
[{:customer/referrals
[:customer/name]}]}
{:>/phones
[{:customer/phones
[:phone/id
:phone/number]}]}]}])
(defn run-benchmark [runners {:keys [entity eql]}]
(bench->chart runners entity eql))
(defn run-benchmarks [runners benchmarks]
(doseq [{:keys [name] :as bench} benchmarks]
(let [results (run-benchmark runners bench)]
(println name)
(println
(json/generate-string results {:pretty true})))))
(comment
; run all the benchmarks
(run-benchmarks all-runners benchmarks))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment