Datomic queries against Clojure collections

gistfile1.clj

;; Datomic example code
(use '[datomic.api :only (db q) :as d])

;; ?answer binds a scalar
(q '[:find ?answer :in ?answer]
   42)

;; of course you can bind more than one of anything
(q '[:find ?last ?first :in ?last ?first]
   "Doe" "John")

;; [?last ?first] binds a tuple
(q '[:find ?last ?first :in [?last ?first]]
   ["Doe" "John"])

;; [?first ...] binds a collection
(q '[:find ?first
     :in [?first ...]]
   ["John" "Jane" "Phineas"])

;; [[?first ?last]] binds a relation
(q '[:find ?first
     :in [[?first ?last]]]
   [["John" "Doe"]
    ["Jane" "Doe"]])

;; a database binding name starts with $ instead of ?
;; any relation with 4-tuples E/A/V/T can act as a database
;; so in Datomic, you can mock a database with a list of lists
(q '[:find ?first
     :in $db
     :where [$db _ :firstName ?first]]
   [[1 :firstName "John"]])

;; same as previous, but omit $db for single-database query
;; any relation with 4-tuples eavt can act as a database
(q '[:find ?first
     :where [_ :firstName ?first]]
   [[1 :firstName "John" 42]
    [1 :lastName "Doe" 42]])

;; simple in-memory join, two tuple bindings
(q '[:find ?first ?height
     :in [?last ?first ?email] [?email ?height]]
   ["Doe" "John" "jdoe@example.com"]
   ["jdoe@example.com" 71])

;; simple in-memory join, two relation bindings
;; see next example for a faster approach
(q '[:find ?first ?height
     :in [[?last ?first ?email]] [[?email ?height]]]
   [["Doe" "John" "jdoe@example.com"]
    ["Doe" "Jane" "jane@example.com"]]
   [["jane@example.com" 73]
    ["jdoe@example.com" 71]])

;; same as previous example, but with database expressions
;; runs faster than relation bindings (as of July 2012) 
(q '[:find ?first ?height
     :in $a $b
     :where [$a ?last ?first ?email]
            [$b ?email ?height]]
   [["Doe" "John" "jdoe@example.com"]
    ["Doe" "Jane" "jane@example.com"]]
   [["jane@example.com" 73]
    ["jdoe@example.com" 71]])

;; simple in-memory join, two database bindings
(q '[:find ?first ?height
     :in $db1 $db2
     :where [$db1 ?e1 :firstName ?first]
            [$db1 ?e1 :email ?email]
            [$db2 ?e2 :email ?email]
            [$db2 ?e2 :height ?height]]
   [[1 :firstName "John"]
    [1 :email  "jdoe@example.com"]
    [2 :firstName "Jane"]
    [2 :email "jane@example.com"]]
   [[100 :email "jane@example.com"]
    [100 :height 73]
    [101 :email "jdoe@example.com"]
    [101 :height 71]])

;; compare to http://stackoverflow.com/questions/3717939/iterating-and-processing-an-arraylist
(q '[:find ?car ?speed
     :in [[?car ?speed]]
     :where [(> ?speed 100)]]
   [["Stock" 225]
    ["Spud" 80]
    ["Rocket" 400]
    ["Stock" 225]
    ["Clunker" 40]])

;; compare to http://stackoverflow.com/questions/109383/how-to-sort-a-mapkey-value-on-the-values-in-java
(->> (q '[:find ?k ?v
          :in [[?k ?v] ...]]
        {:D 67.3 :A 99.5 :B 67.4 :C 67.5})
     (sort-by second))

The age of the question-mark prefixed symbol in Clojure is nigh!

I like the idea of using datomic queries over clojure collections.
I am not sure how to use it to solve my problem.
I am not sure if I am approaching this problem correctly/appropriately.

Example:
(def collection [{:key-1 :val-a1 :key-2 :val-a2 ... :key-100 :val-a100}
{:key-1 :val-b1 :key-2 :val-b2 ... :key-100 :val-b100}
...
{:key-1 :val-x1 :key-2 :val-x2 ... :key-100 :val-x100}
{:key-1 :val-z1 :key-2 :val-z2 ... :key-100 :val-z100}])

(defn find-all
"Return the hash where the value for :key-99 is equal to :val-x100."
[collection]
(d/q '[:find ?c
:in [?c ...]
:where
[(-> ?c :key-100 (= :val-x100)) ?f]
[(= ?f true)]]
collection))

(find-all collection)

{[{:key-100 :val-x100, :key-1 :val-x1, :key-2 :val-x2}]}

;; I need to parameterize the value and pass it in as an argument.
;; Currently this does not work.

(defn find-all2
"Return the hash where the value for :key-99 is equal to 'value'."
[value collection]
(d/q '[:find ?c
:in ?v [?c ...]
:where
[(-> ?c :key-100 (= ?v)) ?f]
[(= ?f true)]]
value collection))

(find-all2 :val-x100 collection)
CompilerException java.lang.RuntimeException: Unable to resolve symbol: ?v in this context, compiling:(/private/var/folders/0f/4w3qyc8s5vs9bnv6lfh5pg_80000gn/T/form-init8853993715126212403.clj:7:64)

@stuarthalloway

Has there been any discussion about extending datomic.client.api/q to allow this sort of query behavior in Datomic Cloud?

(d/q '[:find ?first ?height
       :in $a $b
       :where [$a ?last ?first ?email]
       [$b ?email ?height]]
     [["Doe" "John" "jdoe@example.com"]
      ["Doe" "Jane" "jane@example.com"]]
     [["jane@example.com" 73]
      ["jdoe@example.com" 71]])
Execution error (ExceptionInfo) at datomic.client.api.impl/incorrect (impl.clj:42).
Query args must include a database

Looks like collections don't satisfy the Queryable protocol and thus fail.

Now, that we have in-memory dev-local db, we can whip up a dummy db for these kinds of queries:

  (let [dummy-opts {:server-type :dev-local
                    :storage-dir :mem
                    :system      "dummy"
                    :db-name     "dummy"}
        dummy-db (-> dummy-opts
                     d/client
                     (doto (d/create-database dummy-opts))
                     (d/connect dummy-opts)
                     d/db)]
    (d/q '[:find ?first ?height
           :in $ $a $b
           :where [$a ?last ?first ?email]
           [$b ?email ?height]]
         dummy-db
         [["Doe" "John" "jdoe@example.com"]
          ["Doe" "Jane" "jane@example.com"]]
         [["jane@example.com" 73]
          ["jdoe@example.com" 71]]))

or maybe this is cleaner, and also makes it easy to reuse the dummy db:

  (def dummy-datomic-system
    {:server-type :dev-local
     :storage-dir :mem
     :system      "dummy-system"})

  (def dummy-db-ref
    {:db-name "dummy-db"})

  (def dummy-db
    (-> dummy-datomic-system
        d/client
        (doto (d/create-database dummy-db-ref))
        (d/connect dummy-db-ref)
        d/db))

  (d/q '[:find ?first ?height
         :in $ $a $b
         :where [$a ?last ?first ?email]
         [$b ?email ?height]]
       dummy-db
       [["Doe" "John" "jdoe@example.com"]
        ["Doe" "Jane" "jane@example.com"]]
       [["jane@example.com" 73]
        ["jdoe@example.com" 71]])

if we use map-args with d/q, then the query would look like this

  (d/q {:query '[:find ?first ?height
                 :in $ $a $b
                 :where [$a ?last ?first ?email]
                 [$b ?email ?height]]
        :args  [dummy-db
                [["Doe" "John" "jdoe@example.com"]
                 ["Doe" "Jane" "jane@example.com"]]
                [["jane@example.com" 73]
                 ["jdoe@example.com" 71]]]})

as we can see, the dummy-db and the :in $ goes hand in hand, so we could write a function, which just adds those to queries.
if the query would be in map format, then such function would be easier to implement:

  (def query-expecting-a-db
    '{:find  [?first ?height]
      :in    [$ $a $b]
      :where [[$a ?last ?first ?email]
              [$b ?email ?height]]})
  
  (d/q {:query query-expecting-a-db
        :args  [dummy-db
                [["Doe" "John" "jdoe@example.com"]
                 ["Doe" "Jane" "jane@example.com"]]
                [["jane@example.com" 73]
                 ["jdoe@example.com" 71]]]})

then we can write a function, which injects the dummy-db and doctors the :in clause:

  (def query-non-db-data
    '{:find  [?first ?height]
      :in    [$a $b]
      :where [[$a ?last ?first ?email]
              [$b ?email ?height]]})

  (defn with-dummy-db [q & args]
    {:query (-> q (update :in #(into '[$] %)))
     :args  (into [dummy-db] args)})

  (-> query-non-db-data
      (with-dummy-db
        [["Doe" "John" "jdoe@example.com"]
         ["Doe" "Jane" "jane@example.com"]]
        [["jane@example.com" 73]
         ["jdoe@example.com" 71]])
      d/q)