the not-very-lazy ResultSet

result-set-test.clj

(ns result-set-test.core
  (:require [clojure.java.jdbc :as jdbc]
            [clojure.string :as str])
  (:gen-class)
  (:import (java.sql ResultSet)
           (clojure.lang IReduceInit)))

(def sqlvec ["select left(md5(i::text), 10),
            md5(random()::text),
            md5(random()::text),
            left(md5(random()::text), 4)
            from generate_series(1, 100000000) s(i)"])

(def db-spec {:connection-uri "jdbc:postgresql://localhost:5432/postgres?user=postgres"})

(defn test1 []
  (let [r (fn result-set-fn [rs]
            (doseq [row rs]
              nil))]

    (jdbc/with-db-transaction [db db-spec]
      (jdbc/query db sqlvec {:concurrency   :read-only
                             :fetch-size    1000
                             :result-type   :forward-only
                             :result-set-fn r}))))

(defn reducible-result-set
  "https://gist.github.com/ghadishayban/d2f31961deba98ee4595"
  [^ResultSet rs]
  (reify IReduceInit
    (reduce [this f init]
      (let [rsmeta (.getMetaData rs)
            idxs   (range 1 (inc (.getColumnCount rsmeta)))

            col    (fn [^Integer i]
                     (.getColumnLabel rsmeta i))

            val    (fn [^Integer i]
                     (-> rs
                         (.getObject i)
                         (jdbc/result-set-read-column rsmeta i)))

            keys   (->> (map col idxs)
                        (#'jdbc/make-cols-unique)
                        (map (comp keyword str/lower-case)))]
        (loop [ret init]
          (if (.next rs)
            (let [ret (f ret (zipmap keys (map val idxs)))]
              (if (reduced? ret)
                @ret
                (recur ret)))
            ret))))))

(defn test2 []
  (let [use-result-set (fn use-result-set [rs]
                         (into [] (comp (map (fn [x] nil))
                                        (remove some?)) rs))
        result-set-fn  (fn result-set-fn [rs]
                         (use-result-set (reducible-result-set rs)))]
    (jdbc/with-db-transaction [db db-spec]
      (jdbc/db-query-with-resultset db sqlvec result-set-fn
                                    {:concurrency :read-only
                                     :fetch-size  1000
                                     :result-type :forward-only}))))

(comment 
  :dependencies 
   [[org.clojure/clojure "1.9.0-alpha17"]
   [org.clojure/java.jdbc "0.7.0-alpha3"]
   [org.postgresql/postgresql "42.1.1.jre7"]]

:jvm-opts ["-Duser.timezone=UTC"
             "-server"
             "-d64"
             "-Xms1g"
             "-Xmx1g"
             "-Xss512k"
             "-XX:+UseConcMarkSweepGC"
             "-XX:+CMSParallelRemarkEnabled"
             "-XX:+UseCMSInitiatingOccupancyOnly"
             "-XX:CMSInitiatingOccupancyFraction=-1"
             "-XX:CMSTriggerRatio=50"
             "-XX:NewRatio=3"
             "-XX:+UseParNewGC"
             "-XX:SurvivorRatio=6"
             "-XX:MaxTenuringThreshold=1"
             "-XX:-OmitStackTraceInFastThrow"]
  )