add-vs-from-private-data-store.clj

;; install schema
[{:db/ident :myapp.migrations/write-PrivateDataStore-k-if-still-valid
  :db/dow "Adds the [e new-attr new-attr-v] datom if and only if e does not already have a value for new-attr. Note that new-attr should be :db.cardinality/one"
  :db/fn #db/fn {:lang "clojure"
                 :require [[datomic.api :as d]]
                 :params [db e new-attr new-attr-v]
                 :code 
                 (when (empty? 
                         ;; NOTE choose the index to look up in accordance with your imports - see Note (1) below
                         (d/datoms db :eavt e new-attr))
                   [:db/add e new-attr new-attr-v])}}]
                   
;; usage: 
[[:myapp.migrations/write-PrivateDataStore-k-if-still-valid
  100 :user/email--k #uuid"fb23991a-d7c7-4850-9735-904345325281"]
 [:myapp.migrations/write-PrivateDataStore-k-if-still-valid
  100 :user/first-name--k #uuid"e6f7ac4e-70a3-4427-9d5a-93488adc134a"]
 [:myapp.migrations/write-PrivateDataStore-k-if-still-valid
  100 :user/last-name--k #uuid"348f0967-c2d5-45d5-8dbc-a562f75bbbd6"]]

;;;; OR if you're using Datofu, you won't need to install a new database function at all:
;;;; see https://github.com/vvvvalvalval/datofu#writing-migrations-in-datalog

[[:datofu.utils/datalog-add
  [:find ?e ?new-attr ?new-attr-v 
    :in $ [[?e ?old-attr ?new-attr ?new-attr-v]] 
    :where
    [?e ?old-attr]
    (not [?e ?new-attr])]
  [[100 :user/email :user/email--k #uuid"fb23991a-d7c7-4850-9735-904345325281"]
   [100 :user/first-name :user/first-name--k #uuid"e6f7ac4e-70a3-4427-9d5a-93488adc134a"]
   [100 :user/last-name :user/last-name--k #uuid"348f0967-c2d5-45d5-8dbc-a562f75bbbd6"]]]]


;;;; (1) NOTE: choosing the index in accordance with your imports
;; When importing a large number datoms via :myapp.migrations/write-PrivateDataStore-k-if-still-valid,
;; you want to make sure that the (d/datoms ...) lookup will be done efficiently, i.e hits the Object cache 
;; frequently, otherwise your import process will be drastically slowed down by a lot of segment-fetching
;; roundtrips to storage.
;; One way to achieve this is to have your imports be sorted in a way that matches the used index.
;; For instance, since our :myapp.migrations/write-PrivateDataStore-k-if-still-valid uses and EAVT lookup,
;; then the clauses in our imports should be sorted by EAVT as well.
;; I have observed a 30X performance improvement in import throughput by using this optimization - not to mention 
;; the economy in resource utilization.
;; Tip: one easy way to sort is to put all your [:myapp.migrations/write-PrivateDataStore-k-if-still-valid ...]
;; import clauses in one giant EDN file, one clause per line, and to use the `sort` Unix tool.