deps.edn

{:deps  {com.wsscode/async   {:mvn/version "1.0.11"}
         com.wsscode/pathom  {:mvn/version "2.4.0"}
         com.wsscode/pathom3 {:mvn/version "2022.03.04-alpha"}
         criterium/criterium {:mvn/version "0.4.6"}
         cheshire/cheshire   {:mvn/version "5.10.1"}
         org.clojure/clojure {:mvn/version "1.11.0-rc1"}}}

pathom-benchmarks.clj

(ns com.wsscode.blog.benchmark
  (:require
    [cheshire.core :as json]
    [clojure.core.async]
    [clojure.set :as set]
    [com.wsscode.async.async-clj :as async]
    [com.wsscode.misc.coll :as coll]
    [com.wsscode.misc.time :as time]
    [com.wsscode.pathom.connect :as pc]
    [com.wsscode.pathom.core :as p]
    [com.wsscode.pathom3.connect.built-in.resolvers :as pbir]
    [com.wsscode.pathom3.connect.indexes :as pci]
    [com.wsscode.pathom3.connect.operation :as pco]
    [com.wsscode.pathom3.connect.runner.async :as pcra]
    [com.wsscode.pathom3.interface.async.eql :as p.a.eql]
    [com.wsscode.pathom3.interface.eql :as p.eql]
    [com.wsscode.promesa.bridges.core-async]
    [criterium.core :as c]
    [promesa.core :as pp]))

(defn format-time [mean]
  (let [[factor unit] (c/scale-time mean)]
    (c/format-value mean factor unit)))

(defn bench
  "Runs `f` for `times` times, returns the average time to run it.
  Example:

      (bench 10000 #(do-something))"
  [{:keys [name runner] :as b} entity tx]
  (println name)
  (let [r       (runner entity tx)
        results (c/quick-benchmark (r) {})
        mean    (-> results :sample-mean first)
        mean-ms (* (double mean) 1000.0)]
    (println (str mean-ms "ms"))
    (-> b
        (assoc
          :sample-mean
          mean-ms

          :formatted-mean
          (format-time mean))
        (dissoc :runner))))

(defn bench-compare
  "Compute multiple benchmarks together with names. Example:

      (bench-compare 10000
        {:op-a #(do-something)
         :other-op #(do-something-else)})"
  [runners entity tx]
  (let [data (into [] (map #(bench % entity tx)) runners)
        {:keys [sample-mean]} (apply min-key :sample-mean data)]
    (mapv #(assoc % :variance (/ (:sample-mean %) sample-mean)) data)))

(defn bench->chart [runners entity tx]
  (->> (bench-compare runners entity tx)
       (into {}
             (map (juxt :id :sample-mean)))))

; region env setup

(defn slow-param-transform [config]
  (let [k (if (contains? config ::pco/resolve)
            ::pco/resolve
            ::pc/resolve)]
    (update config k
      (fn [resolve]
        (fn [env input]
          (when-let [delay (-> (or (-> env :ast :params)
                                   (pco/params env))
                               :delay)]
            (time/sleep-ms delay))
          (resolve env input))))))

(defn items-resolver [count]
  (let [kw (keyword (str "items-" count))]
    (pco/resolver (symbol (str "items-" count))
      {::pco/output [{kw [:id]}]
       ::pco/cache? false}
      (fn [{:keys [n]} _]
        {kw (mapv (fn [_] (hash-map :id (vswap! n inc))) (range count))}))))

(defn gen-resolvers [attr]
  (let [attr-s (name attr)]
    [(pco/resolver (symbol attr-s)
       {::pco/input     [:id]
        ::pco/output    [attr]
        ::pco/transform slow-param-transform}
       (fn [_ {:keys [id]}] {attr (* 10 id)}))

     (let [batch-attr (keyword (str attr-s "-batch"))]
       (pco/resolver (symbol (str attr-s "-batch"))
         {::pco/input     [:id]
          ::pco/output    [batch-attr]
          ::pco/batch?    true
          ::pco/transform slow-param-transform}
         (fn [_ ids]
           (mapv #(hash-map batch-attr (* 10 (:id %))) ids))))]))

(pco/defresolver customer-account [{:customer/keys [id]}]
  {:account/id id})

(pco/defresolver customer-info [{:customer/keys [id]}]
  {:customer/name     "Name"
   :customer/birthday "Bday"
   :customer/genre    "Genre"})

(pco/defresolver purchases [{:customer/keys [id]}]
  {::pco/output
   [{:customer/purchases
     [:purchase/id
      :purchase/amount
      :purchase/title]}]}
  {:customer/purchases
   (mapv
     #(array-map
        :purchase/id %
        :purchase/vendor (str "V" %)
        :purchase/amount (rand-int 100))
     (range 50))})

(pco/defresolver purchases-total [{:keys [customer/purchases]}]
  {::pco/input
   [{:customer/purchases
     [:purchase/amount
      :purchase/category]}]}
  {:customer/purchases-total (transduce (map :purchase/amount) + purchases)})

(pco/defresolver chargebacks [{:keys [customer/id]}]
  {:customer/chargebacks
   [{:chargeback/date   "date"
     :chargeback/status "sts"}
    {:chargeback/date   "date"
     :chargeback/status "sts"}
    {:chargeback/date   "date"
     :chargeback/status "sts"}
    {:chargeback/date   "date"
     :chargeback/status "sts"}
    {:chargeback/date   "date"
     :chargeback/status "sts"}
    {:chargeback/date   "date"
     :chargeback/status "sts"}]})

(pco/defresolver boletos [{:keys [customer/id]}]
  {:customer/boletos
   [{:boleto/id     1
     :boleto/number "date"
     :boleto/paid?  "sts"}
    {:boleto/id     2
     :boleto/number "date"
     :boleto/paid?  "sts"}
    {:boleto/id     3
     :boleto/number "date"
     :boleto/paid?  "sts"}]})

(pco/defresolver credit-cards [{:keys [account/id]}]
  {:account/credit-cards
   [{:credit-card/id            1
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}
    {:credit-card/id            2
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}
    {:credit-card/id            3
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}
    {:credit-card/id            4
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}
    {:credit-card/id            5
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}
    {:credit-card/id            6
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}
    {:credit-card/id            7
     :credit-card/masked-number "0321xxxxxxxxxxx3214"
     :credit-card/status        :credit-card.status/active}]})

(pco/defresolver purchase-category [inputs]
  {::pco/input
   [:purchase/id]

   ::pco/output
   [:purchase/category]

   ::pco/batch? true}
  (mapv #(assoc % :purchase/category (str "cat-" (rand-int 10))) inputs))

(pco/defresolver account-details [{:account/keys [id]}]
  {:account/balance         "balance"
   :account/balance-future  "future"
   :account/onboarding-date "date"
   :account/status          "status"})

(pco/defresolver account-status-history [{:account/keys [id]}]
  {:account/status-history
   [{:status "on"
     :at     "date"}
    {:status "off"
     :at     "date"}
    {:status "on"
     :at     "date"}
    {:status "off"
     :at     "date"}
    {:status "on"
     :at     "date"}]})

(pco/defresolver customer-referrals [{:keys [customer/id]}]
  {:customer/referrals
   [{:customer/id 2}
    {:customer/id 3}
    {:customer/id 4}
    {:customer/id 5}]})

(pco/defresolver customer-phones [{:keys [customer/id]}]
  {:customer/phones
   [{:phone/id 2
     :phone/number "2312424141"}
    {:phone/id 3
     :phone/number "531541241"}]})

(defn delay-resolver [resolver delay-ms]
  (pco/wrap-resolve resolver
    (fn [resolve]
      (fn [env input]
        (if (::pcra/async-runner? env)
          (if (::p/path env)
            (async/go-promise
              (async/<! (clojure.core.async/timeout delay-ms))
              (async/<?maybe (resolve env input)))
            (pp/do!
              (pp/delay delay-ms)
              (resolve env input)))
          (do
            (Thread/sleep delay-ms)
            (resolve env input)))))))

(def registry
  [(items-resolver 1)
   (items-resolver 10)
   (items-resolver 100)
   (items-resolver 1000)
   (items-resolver 10000)
   (gen-resolvers :x)
   (gen-resolvers :y)

   (delay-resolver customer-account 50)
   (delay-resolver customer-info 50)
   (delay-resolver purchases 50)
   (delay-resolver purchase-category 50)
   (delay-resolver account-details 50)
   (delay-resolver account-status-history 50)
   (delay-resolver chargebacks 50)
   (delay-resolver credit-cards 50)
   (delay-resolver boletos 50)
   (delay-resolver customer-referrals 50)
   (delay-resolver customer-phones 50)

   purchases-total

   ; complex route
   (pbir/constantly-resolver :a 1)
   (pbir/single-attr-resolver :g :b inc)
   (pbir/constantly-resolver :c 2)
   (pbir/constantly-resolver :e 1)
   (update (pbir/constantly-resolver :e 3) :config assoc ::pco/op-name 'e1)
   (pbir/single-attr-resolver :e :f inc)
   (update (pbir/constantly-resolver :g 4) :config assoc ::pco/input [:c :f])
   (update (pbir/constantly-resolver :h 5) :config assoc ::pco/input [:a :b])])

(def env-indexes
  (pci/register registry))

(defn base-env [plan-cache]
  (-> env-indexes
      (assoc :n (volatile! 0))
      (cond->
        plan-cache
        (assoc :com.wsscode.pathom3.connect.planner/plan-cache* plan-cache))
      ((requiring-resolve 'com.wsscode.pathom.viz.ws-connector.pathom3/connect-env)
       "debug")))

(defn resolver-p3->p2 [resolver]
  (let [{::pc/keys [transform] :as config} (pco/operation-config resolver)]
    (-> config
        (set/rename-keys {::pco/op-name ::pc/sym
                          ::pco/input   ::pc/input
                          ::pco/output  ::pc/output
                          ::pco/batch?  ::pc/batch?
                          ::pco/cache?  ::pc/cache})
        (assoc ::pc/resolve (:resolve resolver))
        (coll/update-if ::pc/input set)
        (cond-> transform transform))))

(def p2-registry
  (mapv resolver-p3->p2 (flatten registry)))

; endregion

; region runners

(defn run-p2-serial [entity query]
  (let [parser
        (p/parser
          {::p/env     {::p/reader               [p/map-reader
                                                  pc/reader2
                                                  pc/open-ident-reader
                                                  p/env-placeholder-reader]
                        ::p/placeholder-prefixes #{">"}}
           ::p/mutate  pc/mutate
           ::p/plugins [(pc/connect-plugin {::pc/register p2-registry})
                        p/error-handler-plugin]})]
    #(parser {:n         (volatile! 0)
              ::p/entity (atom entity)}
       query)))

(defn run-p2-async [entity query]
  (let [parser
        (p/async-parser
          {::p/env     {::p/reader               [p/map-reader
                                                  pc/async-reader2
                                                  pc/open-ident-reader
                                                  p/env-placeholder-reader]
                        ::pcra/async-runner?     true
                        ::p/placeholder-prefixes #{">"}}
           ::p/mutate  pc/mutate-async
           ::p/plugins [(pc/connect-plugin {::pc/register p2-registry})
                        p/error-handler-plugin]})]
    #(-> (parser {:n         (volatile! 0)
                  ::p/entity (atom entity)}
           query)
         promesa.core/promise
         (deref))))

(defn run-p2-parallel [entity query]
  (let [parser
        (p/parallel-parser
          {::p/env     {::p/reader               [p/map-reader
                                                  pc/parallel-reader
                                                  pc/open-ident-reader
                                                  p/env-placeholder-reader]
                        ::pcra/async-runner?     true
                        ::p/placeholder-prefixes #{">"}}
           ::p/mutate  pc/mutate-async
           ::p/plugins [(pc/connect-plugin {::pc/register p2-registry})
                        p/error-handler-plugin]})]
    #(-> (parser {:n         (volatile! 0)
                  ::p/entity (atom entity)}
           query)
         promesa.core/promise
         (deref))))

(defn run-p3-serial [entity query]
  #(p.eql/process (base-env nil) entity query))

(defn run-p3-async [entity query]
  #(deref (p.a.eql/process (base-env nil) entity query)))

(defn run-p3-parallel [entity query]
  #(deref (p.a.eql/process (assoc (base-env nil) ::p.a.eql/parallel? true) entity query)))

(defn run-p3-serial-cached [entity query]
  (let [cache (atom {})
        env   (base-env cache)]
    (p.eql/process env entity query)
    #(p.eql/process env entity query)))

(defn run-p3-async-cached [entity query]
  (let [cache (atom {})
        env   (base-env cache)]
    (deref (p.a.eql/process env entity query))
    #(deref (p.a.eql/process env entity query))))

(defn run-p3-parallel-cached [entity query]
  (let [cache (atom {})
        env   (assoc (base-env cache) ::p.a.eql/parallel? true)]
    (deref (p.a.eql/process env entity query))
    #(deref (p.a.eql/process env entity query))))

(def all-runners
  [{:name   "Pathom 3 Serial Cached Plan"
    :id     "p3-serial-cp"
    :runner run-p3-serial-cached}
   {:name   "Pathom 3 Serial"
    :id     "p3-serial"
    :runner run-p3-serial}
   {:name   "Pathom 3 Async Cached Plan"
    :id     "p3-async-cp"
    :runner run-p3-async-cached}
   {:name   "Pathom 3 Async"
    :id     "p3-async"
    :runner run-p3-async}
   {:name   "Pathom 3 Parallel Cached Plan"
    :id     "p3-parallel-cp"
    :runner run-p3-parallel-cached}
   {:name   "Pathom 3 Parallel"
    :id     "p3-parallel"
    :runner run-p3-parallel}
   {:name   "Pathom 2 Serial"
    :id     "p2-serial"
    :runner run-p2-serial}
   {:name   "Pathom 2 Async"
    :id     "p2-async"
    :runner run-p2-async}
   {:name   "Pathom 2 Parallel"
    :id     "p2-parallel"
    :runner run-p2-parallel}])

; endregion

(def benchmarks
  [{:name   "Read a single attribute"
    :entity {:x 1}
    :eql    [:x]}

   {:name   "Complex plan"
    :entity {:x 1}
    :eql    [:h]}

   {:name   "1000 items sequence"
    :entity {}
    :eql    [{:items-1000 [:x]}]}

   {:name   "1000 items in batch"
    :entity {}
    :eql    [{:items-1000 [:x-batch]}]}

   {:name   "10 items with 10 children with 10 children"
    :entity {}
    :eql    [{:items-10
              [:x
               {:items-10
                [:y
                 {:items-10
                  [:x]}]}]}]}

   {:name   "100 items with 10 children, batch head"
    :entity {}
    :eql    [{:items-100
              [:x-batch
               {:items-10
                [:y]}]}]}

   {:name   "100 items with 10 children, batch tail"
    :entity {}
    :eql    [{:items-100
              [:y
               {:items-10
                [:x-batch]}]}]}

   {:name   "100 items, delay 200 batch"
    :entity {}
    :eql    '[{:items-100
               [(:x-batch {:delay 200})]}]}

   {:name   "10 items with 10 children, delay 20, batch"
    :entity {}
    :eql    '[{:items-10
               [{:items-10
                 [(:x-batch {:delay 20})]}]}]}

   {:name   "Many widgets"
    :entity {:customer/id 1}
    :eql    [{:>/profile-info
              [:customer/name
               :customer/birthday
               :customer/genre
               :account/status]}
             {:>/purchases
              [{:customer/purchases
                [:purchase/vendor
                 :purchase/amount
                 :purchase/category]}]}
             {:>/account-info
              [:account/status
               :account/status-history
               :account/onboarding-date
               :account/balance
               :account/balance-future]}
             {:>/chargebacks
              [{:customer/chargebacks
                [:chargeback/date
                 :chargeback/status]}]}
             {:>/credit-cards
              [{:account/credit-cards
                [:credit-card/masked-number
                 :credit-card/status]}]}
             {:>/boletos
              [{:customer/boletos
                [:boleto/number
                 :boleto/paid?]}]}
             {:>/mgm
              [{:customer/referrals
                [:customer/name]}]}
             {:>/phones
              [{:customer/phones
                [:phone/id
                 :phone/number]}]}]}])

(defn run-benchmark [runners {:keys [entity eql]}]
  (bench->chart runners entity eql))

(defn run-benchmarks [runners benchmarks]
  (doseq [{:keys [name] :as bench} benchmarks]
    (let [results (run-benchmark runners bench)]
      (println name)
      (println
        (json/generate-string results {:pretty true})))))

(comment
  ; run all the benchmarks
  (run-benchmarks all-runners benchmarks))