Dash xodus wrapper

core.clj

(ns nl.avisi.xodus.core
  (:import [jetbrains.exodus.entitystore PersistentEntityStore StoreTransaction Entity PersistentEntityStoreImpl StoreTransactionalComputable EntityIterable]
           [jetbrains.exodus.entitystore.iterate SortIterable])
  (:require [clojure.tools.logging :as log]))

(defn with-transaction!
  "Calls the given function with a transaction.
  Will revert all changes on exception and throw the exception"
  ([^PersistentEntityStore store func]
   (with-transaction! store func false))
  ([^PersistentEntityStore store func read-only?]
   (let [computable (reify StoreTransactionalComputable
                      (compute [this ^StoreTransaction tx]
                        (func tx)))]
     (if read-only?
       (.computeInReadonlyTransaction store computable)
       (.computeInTransaction store computable)))))

(defn set-props! [^Entity entity props]
  (dorun
   (map
    #(.setProperty entity (name %) (get props %))
    (keys props))))

(defn flush-tx [^StoreTransaction tx]
  (.flush tx))

(defn map->entity ^Entity [^StoreTransaction transaction entity datum]
  (let [ent ^Entity (.newEntity transaction entity)]
    (set-props! ent datum)
    ent))

(defn all [^StoreTransaction transaction entity]
  (seq (.getAll transaction entity)))

(defn all-by-property [^StoreTransaction transaction entity property value]
  (seq (.find transaction entity (name property) value)))

(defn all-by-props
  "Finds an entity by multiple values props-with-vals-tuple should be like [[:prop 1] [:prop2 10]"
  [^StoreTransaction transaction entity props-with-vals-tuple]
  (let [queries (map (fn [[prop value]]
                       (.find transaction entity (name prop) value)) props-with-vals-tuple)]
    (seq (reduce (fn [^EntityIterable left ^EntityIterable right]
                   (.intersect left right)) (first queries) (rest queries)))))

(defn entities [^StoreTransaction transaction]
  (.getEntityTypes transaction))

(defn all-in-range [^StoreTransaction transaction entity prop inclusive-start inclusive-end]
  (seq (.find transaction entity (name prop) inclusive-start inclusive-end)))

(defn all-in-range-sorted [^StoreTransaction transaction entity prop inclusive-start inclusive-end sort-by ascending?]
  (seq (.sort
        transaction
        entity
        (name sort-by)
        (.find transaction entity (name prop) inclusive-start inclusive-end)
        ascending?)))

(defn all-sorted-by [^StoreTransaction transaction entity prop ascending?]
  (seq (.sort transaction entity (name prop) ascending?)))

(defn id [^Entity entity]
  (.toIdString entity))

(defn delete [^Entity entity]
  (.delete entity))

(defn save!
  [^StoreTransaction transaction entity datum]
  (let [new-ent ^Entity (map->entity transaction entity datum)]
    (.saveEntity transaction new-ent)
    new-ent)
  )

(defn save-bulk!
  "A shortcut to save a bulk of items, flushes the transaction every 1000 items
  for awesome performance"
  [^StoreTransaction transaction entity data]
  (dorun
   (map-indexed
    (fn [i datum]
      (map->entity transaction entity datum)
      (when (= 0 (mod i 1000)) (.flush transaction)))
    data)))

core_test.clj

(ns nl.avisi.xodus.core-test
  (:import [jetbrains.exodus.entitystore PersistentEntityStores])
  (:require [nl.avisi.xodus.core :refer :all]
            [clojure.test :refer :all]
            [nl.avisi.fixtures.xodus :as xodus]))


(use-fixtures :each xodus/fixture)

(deftest save-single-items
  (testing "Saving a single item"
    (with-transaction! xodus/store
      (fn [tx]
        (let [res (save! tx "test" {"id" 1 "value" 2})]
          (is (= 1 (.getProperty res "id")))
          (is (= 2 (.getProperty res "value"))))))))

(deftest save-and-get-all-items
  (testing "Saving 10 items"
    (with-transaction! xodus/store
      (fn [tx]
        (let [res (save-bulk! tx "test" (repeat 10 {"id" "name"
                                                    "value" 10}))]
          (is (= 10 (count (all tx "test")))))))))

(deftest get-all-in-range
  (testing "getting one item in range"
    (with-transaction! xodus/store
      (fn [tx]
        (save-bulk! tx "test"
                    [{"value" 10}
                     {"value" 11}
                     {"value" 12}
                     {"value" 13}])
        (is (= 3 (count (all-in-range tx "test" "value" 10 12))))))))

(deftest get-all-in-range-sorted
  (testing "getting one item in range"
    (with-transaction! xodus/store
      (fn [tx]
        (save-bulk! tx "test"
                    [{"value" 12}
                     {"value" 11}
                     {"value" 10}
                     {"value" 13}])
        (is (= 3 (count (all-in-range-sorted tx "test" "value" 10 12 :value true))))
        (is (= '(10 11 12) (map #(.getProperty % "value") (all-in-range-sorted tx "test" "value" 10 12 :value true))))))))

(deftest get-all-entities
  (testing "getting two entities"
    (with-transaction! xodus/store
      (fn [tx]
        (save! tx "foo" {"value" 1})
        (save! tx "bar" {"value" 2})))
    (with-transaction! xodus/store
      (fn [tx]
        (is (some #{"foo"} (entities tx)))
        (is (some #{"bar"} (entities tx)))))))

(deftest get-all-sorted-by
  (testing "getting two entities"
    (with-transaction! xodus/store
      (fn [tx]
        (save! tx "foo" {"value" 1})
        (save! tx "foo" {"value" 2})
        (is (= '(1 2) (map #(.getProperty % "value") (all-sorted-by tx "foo" :value true))))
        (is (= '(2 1) (map #(.getProperty % "value") (all-sorted-by tx "foo" "value" false))))))))

(deftest get-by-test
  (testing "getting something by a property"
    (with-transaction! xodus/store
      (fn [tx]
        (save! tx "Post" {"title" "My first post"})
        (is (= 1 (count (all-by-property tx "Post" "title" "My first post"))))))))

(deftest get-by-multiple-props-test
  (testing "getting something by multiple properties property"
    (with-transaction! xodus/store
      (fn [tx]
        (save! tx "Post" {"title" "My first post" "timestamp" 2})
        (save! tx "Post" {"title" "My first post" "timestamp" 10})
        (let [res (all-by-props tx "Post" [["title" "My first post"] ["timestamp" 10]])]
          (is (= (count res) 1))
          (is (= (.getProperty (first res) "title") "My first post"))
          (is (= (.getProperty (first res) "timestamp") 10)))))))

(deftest save-large-string
  (testing "getting something by a property"
    (with-transaction! xodus/store
      (fn [tx]
        (let [big-string (pr-str (map #(hash-map :value %) (range 10000)))]
          (save! tx "Post" {"big-string" big-string})
          (is (= 1 (count (all-by-property tx "Post" "big-string" big-string)))))))))

fixture.clj

(ns nl.avisi.fixtures.xodus
  (:import [jetbrains.exodus.entitystore PersistentEntityStores]))

(defn delete-recursively [fname]
  (let [func (fn [func f]
               (when (.isDirectory f)
                 (doseq [f2 (.listFiles f)]
                   (func func f2)))
               (clojure.java.io/delete-file f))]
    (func func (clojure.java.io/file fname))))

(def folder "test-data")
(def store nil)

(defn fixture
  [f]
  (alter-var-root
   #'store
   (fn [_]
     (let [store (PersistentEntityStores/newInstance "test-data")
               config (.getConfig store)]
           (.setCachingDisabled config true)
           store)))
  (f)
  (try
    (.close store)
    (catch Throwable e))
  (delete-recursively "test-data"))

service.clj

(ns nl.avisi.xodus.service
  (:import [jetbrains.exodus.entitystore PersistentEntityStores StoreTransaction Entity])
  (:import [jetbrains.exodus.entitystore.management EntityStoreConfig ])
  (:require [clojure.tools.logging :as log]
            [puppetlabs.trapperkeeper.core :as trapperkeeper]
            [puppetlabs.trapperkeeper.services :as tk-services]
            [nl.avisi.xodus.core :as core]))

(defprotocol XodusService
  (with-transaction! [this func read-only?]))

(trapperkeeper/defservice xodus-service
  "Holds a Xodus Persisten Entity Store database"
  XodusService
  [[:ConfigService get-in-config]]
  (start [this context]
         (log/info "Starting Xodus service")
         (let [store (PersistentEntityStores/newInstance
                      (str (get-in-config [:data-directory]) "/xodus"))
               config (.getConfig store)]
           (.setCachingDisabled config true)
           (assoc
            context
            :env
            store)))
  (stop [this context]
        (.close (:env context))
        (log/info "Stopping Xodus service")
        (dissoc context :env))
  (with-transaction! [this func read-only?]
    (core/with-transaction! (:env (tk-services/service-context this))
      func read-only?)))