Tests the memory usage of in-memory Datomic db when excising data

datomic_in_memory_excision_test.clj

(require '[datomic.api :as d])

(def db-uri  "datomic:mem://hello")
(d/create-database db-uri)
(def conn (atom {}))

(reset! conn (d/connect db-uri))

(def schema [{:db/id (d/tempid :db.part/db)
 :db/ident :dummy/content
 :db/valueType :db.type/string
 :db/cardinality :db.cardinality/one
 :db/doc "Something that will fill up memory"
 :db.install/_attribute :db.part/db}])

@(d/transact @conn schema)

;; roughly 4K string
(defn create-4k-string [i j]
  (clojure.string/join nil (conj (repeat 1000 0) "-" i "-" j)))

(defn create-data [batch-index item-index]
  ;; Use unique string so JVM cannot do caching
	{:db/id (d/tempid :db.part/user) :dummy/content (create-4k-string batch-index item-index)})

;; Fill the db so the memory is visually growing
(dotimes [i 100]
	;; Write 40000Kb (+ overhead of datoms) to db
	@(d/transact @conn (map (partial create-data i) (range 10000)))
	(println "Transacted batch" i))

(defn excisions [eids]
  (mapv (fn [eid] {:db/id (d/tempid :db.part/user) :db/excise eid}) eids))

(defn datom-count [db]
 (reduce (fn [acc _] (inc acc)) 0 (d/datoms db :eavt)))

(println (datom-count (d/db @conn)) " left")

;; Clean up in memory DB?
(dotimes [i 1000]
  @(d/transact @conn (excisions (take 10000 (map :e (drop 1000 (d/datoms (d/db conn) :eavt))))))
  (println (datom-count (d/db @conn)) " left")
)
;; Unfortunately the memory keeps on growing and not shrinking

;; We can apply a different trick by just removing the database and references:

;; Keep references to the db
(def db-atom (atom nil))

(reset! db-atom (d/db @conn))
(d/delete-database "datomic:mem://hello")
(reset! conn nil)
(System/gc)
;; Small drop in memory, but db still queryable:

(datom-count @db-atom)

(reset! db-atom nil)
(System/gc)
;; Memory drop? => Yes!

Excision has no effect on the mem database in Datomic as it has no log/storage. See http://docs.datomic.com/excision.html#limitations