Last active
November 11, 2021 02:36
-
-
Save jeroenvandijk/7ebc38218add48ad5ca7 to your computer and use it in GitHub Desktop.
Tests the memory usage of in-memory Datomic db when excising data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(require '[datomic.api :as d]) | |
(def db-uri "datomic:mem://hello") | |
(d/create-database db-uri) | |
(def conn (atom {})) | |
(reset! conn (d/connect db-uri)) | |
(def schema [{:db/id (d/tempid :db.part/db) | |
:db/ident :dummy/content | |
:db/valueType :db.type/string | |
:db/cardinality :db.cardinality/one | |
:db/doc "Something that will fill up memory" | |
:db.install/_attribute :db.part/db}]) | |
@(d/transact @conn schema) | |
;; roughly 4K string | |
(defn create-4k-string [i j] | |
(clojure.string/join nil (conj (repeat 1000 0) "-" i "-" j))) | |
(defn create-data [batch-index item-index] | |
;; Use unique string so JVM cannot do caching | |
{:db/id (d/tempid :db.part/user) :dummy/content (create-4k-string batch-index item-index)}) | |
;; Fill the db so the memory is visually growing | |
(dotimes [i 100] | |
;; Write 40000Kb (+ overhead of datoms) to db | |
@(d/transact @conn (map (partial create-data i) (range 10000))) | |
(println "Transacted batch" i)) | |
(defn excisions [eids] | |
(mapv (fn [eid] {:db/id (d/tempid :db.part/user) :db/excise eid}) eids)) | |
(defn datom-count [db] | |
(reduce (fn [acc _] (inc acc)) 0 (d/datoms db :eavt))) | |
(println (datom-count (d/db @conn)) " left") | |
;; Clean up in memory DB? | |
(dotimes [i 1000] | |
@(d/transact @conn (excisions (take 10000 (map :e (drop 1000 (d/datoms (d/db conn) :eavt)))))) | |
(println (datom-count (d/db @conn)) " left") | |
) | |
;; Unfortunately the memory keeps on growing and not shrinking | |
;; We can apply a different trick by just removing the database and references: | |
;; Keep references to the db | |
(def db-atom (atom nil)) | |
(reset! db-atom (d/db @conn)) | |
(d/delete-database "datomic:mem://hello") | |
(reset! conn nil) | |
(System/gc) | |
;; Small drop in memory, but db still queryable: | |
(datom-count @db-atom) | |
(reset! db-atom nil) | |
(System/gc) | |
;; Memory drop? => Yes! |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Excision has no effect on the mem database in Datomic as it has no log/storage. See http://docs.datomic.com/excision.html#limitations