Skip to content

Instantly share code, notes, and snippets.

@Jaretbinford

Jaretbinford/407-rl-test

Created July 25, 2018 14:45
Show Gist options
  • Save Jaretbinford/d2493053dcb292a1400d4dde4998343f to your computer and use it in GitHub Desktop.
Save Jaretbinford/d2493053dcb292a1400d4dde4998343f to your computer and use it in GitHub Desktop.
Download ZIP
Raw
407-rl-test
{ $.Msg = "ResourceUtilization" }
(ns cloud-tester.core)
(require '[datomic.client.api :as d])
(def cfg {:server-type :cloud
:region "us-east-1" ;; e.g. us-east-1
:system "jaret-lambda-test"
:query-group "jaret-lambda-test"
:endpoint "http://entry.jaret-lambda-test.us-east-1.datomic.net:8182/"
:proxy-port 8182})
;
;(def cfg {:server-type :cloud
; :region "us-east-1" ;; e.g. us-east-1
; :system "jaret-solo-master-july"
; :query-group "jaret-solo-master-july"
; :endpoint "http://entry.jaret-solo-master-july.us-east-1.datomic.net:8182/"
; :proxy-port 8182})
;
;(def client (d/client cfg))
;
;(d/list-databases client {})
;
;(d/create-database client {:db-name "movies"})
;(def conn (d/connect client {:db-name "movies"}))
;
;
(def client (d/client cfg))
(d/list-databases client {})
(def conn (d/connect client {:db-name "mbrainz-subset"}))
(require '[datomic.client.api :as d])
(set! *print-length* 10)
;; get a client and db object of your choice
;; this query just makes memory, does not even look at the db
;; returns a result set as large at ct passed in
(def db (d/db conn))
(def q '[:find ?c
:in $ ?ct
:where [(range ?ct) [?c ...]]])
(time (d/q q db 100000))
(def churn-1
(future
(dotimes [_ 1000]
(Thread/sleep 1000)
(time
(count (d/q q db (* 25 1000)))))))
(def churn-2
(future
(dotimes [_ 100]
(Thread/sleep 10000)
(time
(count (d/q q db (* 25 1000 10)))))))
(def churn-3
(future
(dotimes [_ 10]
(Thread/sleep 10000)
(time
(count (d/q q db (* 25 1000 100)))))))
[ churn-1 churn-2 churn-3]
churn-1
churn-2
churn-3
on old stack churn-3 errors:
#object[clojure.core$future_call$reify__8097
0x2b24da88
{:status :failed,
:val #error{:cause "Specified iterator was dropped",
:data {:cognitect.anomalies/category :cognitect.anomalies/unavailable,
:cognitect.anomalies/message "Specified iterator was dropped",
:dbs [{:database-id "ba146c1e-3d87-49d9-8c6c-f2a2512fdf51",
:t 33,
:next-t 34,
:history false}]},
:via [{:type java.util.concurrent.ExecutionException,
:message "clojure.lang.ExceptionInfo: Specified iterator was dropped {:cognitect.anomalies/category :cognitect.anomalies/unavailable, :cognitect.anomalies/message \"Specified iterator was dropped\", :dbs [{:database-id \"ba146c1e-3d87-49d9-8c6c-f2a2512fdf51\", :t 33, :next-t 34, :history false}]}",
:at [java.util.concurrent.FutureTask report "FutureTask.java" 122]}
{:type clojure.lang.ExceptionInfo,
:message "Specified iterator was dropped",
:data {:cognitect.anomalies/category :cognitect.anomalies/unavailable,
:cognitect.anomalies/message "Specified iterator was dropped",
:dbs [{:database-id "ba146c1e-3d87-49d9-8c6c-f2a2512fdf51",
:t 33,
:next-t 34,
:history false}]},
:at [clojure.core$ex_info invokeStatic "core.clj" 4739]}],
:trace [[clojure.core$ex_info invokeStatic "core.clj" 4739]
[clojure.core$ex_info invoke "core.clj" 4739]
[datomic.client.api.async$ares invokeStatic "async.clj" 56]
[datomic.client.api.async$ares invoke "async.clj" 52]
[datomic.client.api.sync$unchunk invokeStatic "sync.clj" 46]
[datomic.client.api.sync$unchunk invoke "sync.clj" 44]
[datomic.client.api.sync$eval13090$fn__13111 invoke "sync.clj" 99]
[datomic.client.api.impl$eval1716$fn__1717$G__1707__1724 invoke "impl.clj" 28]
[datomic.client.api$q invokeStatic "api.clj" 288]
[datomic.client.api$q invoke "api.clj" 259]
[datomic.client.api$q invokeStatic "api.clj" 291]
[datomic.client.api$q doInvoke "api.clj" 259]
[clojure.lang.RestFn invoke "RestFn.java" 439]
[cloud_tester.core$fn__13673 invokeStatic "core.clj" 67]
[cloud_tester.core$fn__13673 invoke "core.clj" 63]
[clojure.core$binding_conveyor_fn$fn__5476 invoke "core.clj" 2022]
[clojure.lang.AFn call "AFn.java" 18]
[java.util.concurrent.FutureTask run "FutureTask.java" 266]
[java.util.concurrent.ThreadPoolExecutor runWorker "ThreadPoolExecutor.java" 1142]
[java.util.concurrent.ThreadPoolExecutor$Worker run "ThreadPoolExecutor.java" 617]
[java.lang.Thread run "Thread.java" 745]]}}]
on new stack:
;churn-3
;=> #object[clojure.core$future_call$reify__8097 0x33cc4696 {:status :pending, :val nil}]
;churn-2
;=> #object[clojure.core$future_call$reify__8097 0x2ea082e9 {:status :pending, :val nil}]
;"Elapsed time: 3877.982836 msecs"
;churn-1
;=> #object[clojure.core$future_call$reify__8097 0x65d46d6b {:status :pending, :val nil}]
adding more churn caused churn-1 to fall over with query timeout:
=>
#object[clojure.core$future_call$reify__8097
0x65d46d6b
{:status :failed,
:val #error{:cause "processing clause: {:argvars nil, :fn #object[datomic.datalog$expr_clause$fn__23678 0x656117d1 \"datomic.datalog$expr_clause$fn__23678@656117d1\"], :clause [(ground $__in__2) ?ct], :binds [?ct], :bind-type :scalar, :needs-source true}, message: java.util.concurrent.TimeoutException: Query canceled: timeout elapsed",
:data {:cognitect.anomalies/category :cognitect.anomalies/incorrect,
:cognitect.anomalies/message "processing clause: {:argvars nil, :fn #object[datomic.datalog$expr_clause$fn__23678 0x656117d1 \"datomic.datalog$expr_clause$fn__23678@656117d1\"], :clause [(ground $__in__2) ?ct], :binds [?ct], :bind-type :scalar, :needs-source true}, message: java.util.concurrent.TimeoutException: Query canceled: timeout elapsed",
:dbs [{:database-id "ba146c1e-3d87-49d9-8c6c-f2a2512fdf51",
:t 33,
:next-t 34,
:history false}]},
:via [{:type java.util.concurrent.ExecutionException,
:message "clojure.lang.ExceptionInfo: processing clause: {:argvars nil, :fn #object[datomic.datalog$expr_clause$fn__23678 0x656117d1 \"datomic.datalog$expr_clause$fn__23678@656117d1\"], :clause [(ground $__in__2) ?ct], :binds [?ct], :bind-type :scalar, :needs-source true}, message: java.util.concurrent.TimeoutException: Query canceled: timeout elapsed {:cognitect.anomalies/category :cognitect.anomalies/incorrect, :cognitect.anomalies/message \"processing clause: {:argvars nil, :fn #object[datomic.datalog$expr_clause$fn__23678 0x656117d1 \\\"datomic.datalog$expr_clause$fn__23678@656117d1\\\"], :clause [(ground $__in__2) ?ct], :binds [?ct], :bind-type :scalar, :needs-source true}, message: java.util.concurrent.TimeoutException: Query canceled: timeout elapsed\", :dbs [{:database-id \"ba146c1e-3d87-49d9-8c6c-f2a2512fdf51\", :t 33, :next-t 34, :history false}]}",
:at [java.util.concurrent.FutureTask report "FutureTask.java" 122]}
{:type clojure.lang.ExceptionInfo,
:message "processing clause: {:argvars nil, :fn #object[datomic.datalog$expr_clause$fn__23678 0x656117d1 \"datomic.datalog$expr_clause$fn__23678@656117d1\"], :clause [(ground $__in__2) ?ct], :binds [?ct], :bind-type :scalar, :needs-source true}, message: java.util.concurrent.TimeoutException: Query canceled: timeout elapsed",
:data {:cognitect.anomalies/category :cognitect.anomalies/incorrect,
:cognitect.anomalies/message "processing clause: {:argvars nil, :fn #object[datomic.datalog$expr_clause$fn__23678 0x656117d1 \"datomic.datalog$expr_clause$fn__23678@656117d1\"], :clause [(ground $__in__2) ?ct], :binds [?ct], :bind-type :scalar, :needs-source true}, message: java.util.concurrent.TimeoutException: Query canceled: timeout elapsed",
:dbs [{:database-id "ba146c1e-3d87-49d9-8c6c-f2a2512fdf51",
:t 33,
:next-t 34,
:history false}]},
:at [clojure.core$ex_info invokeStatic "core.clj" 4739]}],
:trace [[clojure.core$ex_info invokeStatic "core.clj" 4739]
[clojure.core$ex_info invoke "core.clj" 4739]
[datomic.client.api.async$ares invokeStatic "async.clj" 56]
[datomic.client.api.async$ares invoke "async.clj" 52]
[datomic.client.api.sync$unchunk invokeStatic "sync.clj" 46]
[datomic.client.api.sync$unchunk invoke "sync.clj" 44]
[datomic.client.api.sync$eval13084$fn__13105 invoke "sync.clj" 99]
[datomic.client.api.impl$eval1710$fn__1711$G__1701__1718 invoke "impl.clj" 28]
[datomic.client.api$q invokeStatic "api.clj" 288]
[datomic.client.api$q invoke "api.clj" 259]
[datomic.client.api$q invokeStatic "api.clj" 291]
[datomic.client.api$q doInvoke "api.clj" 259]
[clojure.lang.RestFn invoke "RestFn.java" 439]
[cloud_tester.core$fn__13580 invokeStatic "core.clj" 53]
[cloud_tester.core$fn__13580 invoke "core.clj" 49]
[clojure.core$binding_conveyor_fn$fn__5476 invoke "core.clj" 2022]
[clojure.lang.AFn call "AFn.java" 18]
[java.util.concurrent.FutureTask run "FutureTask.java" 266]
[java.util.concurrent.ThreadPoolExecutor runWorker "ThreadPoolExecutor.java" 1142]
[java.util.concurrent.ThreadPoolExecutor$Worker run "ThreadPoolExecutor.java" 617]
[java.lang.Thread run "Thread.java" 745]]}}]
#object[clojure.core$future_call$reify__8097
0x1b9cf271
{:status :failed,
:val #error{:cause "processing clause: {:argvars (?ct), :fn #object[datomic.extensions$eval208$fn__209 0x13a4d4cb \"datomic.extensions$eval208$fn__209@13a4d4cb\"], :clause [(range ?ct) [?c ...]], :binds [?c], :bind-type :list, :needs-source false}, message: java.lang.OutOfMemoryError: Java heap space",
:data {:cognitect.anomalies/category :cognitect.anomalies/incorrect,
:cognitect.anomalies/message "processing clause: {:argvars (?ct), :fn #object[datomic.extensions$eval208$fn__209 0x13a4d4cb \"datomic.extensions$eval208$fn__209@13a4d4cb\"], :clause [(range ?ct) [?c ...]], :binds [?c], :bind-type :list, :needs-source false}, message: java.lang.OutOfMemoryError: Java heap space",
:dbs [{:database-id "ba146c1e-3d87-49d9-8c6c-f2a2512fdf51",
:t 33,
:next-t 34,
:history false}]},
:via [{:type java.util.concurrent.ExecutionException,
:message "clojure.lang.ExceptionInfo: processing clause: {:argvars (?ct), :fn #object[datomic.extensions$eval208$fn__209 0x13a4d4cb \"datomic.extensions$eval208$fn__209@13a4d4cb\"], :clause [(range ?ct) [?c ...]], :binds [?c], :bind-type :list, :needs-source false}, message: java.lang.OutOfMemoryError: Java heap space {:cognitect.anomalies/category :cognitect.anomalies/incorrect, :cognitect.anomalies/message \"processing clause: {:argvars (?ct), :fn #object[datomic.extensions$eval208$fn__209 0x13a4d4cb \\\"datomic.extensions$eval208$fn__209@13a4d4cb\\\"], :clause [(range ?ct) [?c ...]], :binds [?c], :bind-type :list, :needs-source false}, message: java.lang.OutOfMemoryError: Java heap space\", :dbs [{:database-id \"ba146c1e-3d87-49d9-8c6c-f2a2512fdf51\", :t 33, :next-t 34, :history false}]}",
:at [java.util.concurrent.FutureTask report "FutureTask.java" 122]}
{:type clojure.lang.ExceptionInfo,
:message "processing clause: {:argvars (?ct), :fn #object[datomic.extensions$eval208$fn__209 0x13a4d4cb \"datomic.extensions$eval208$fn__209@13a4d4cb\"], :clause [(range ?ct) [?c ...]], :binds [?c], :bind-type :list, :needs-source false}, message: java.lang.OutOfMemoryError: Java heap space",
:data {:cognitect.anomalies/category :cognitect.anomalies/incorrect,
:cognitect.anomalies/message "processing clause: {:argvars (?ct), :fn #object[datomic.extensions$eval208$fn__209 0x13a4d4cb \"datomic.extensions$eval208$fn__209@13a4d4cb\"], :clause [(range ?ct) [?c ...]], :binds [?c], :bind-type :list, :needs-source false}, message: java.lang.OutOfMemoryError: Java heap space",
:dbs [{:database-id "ba146c1e-3d87-49d9-8c6c-f2a2512fdf51",
:t 33,
:next-t 34,
:history false}]},
:at [clojure.core$ex_info invokeStatic "core.clj" 4739]}],
:trace [[clojure.core$ex_info invokeStatic "core.clj" 4739]
[clojure.core$ex_info invoke "core.clj" 4739]
[datomic.client.api.async$ares invokeStatic "async.clj" 56]
[datomic.client.api.async$ares invoke "async.clj" 52]
[datomic.client.api.sync$unchunk invokeStatic "sync.clj" 46]
[datomic.client.api.sync$unchunk invoke "sync.clj" 44]
[datomic.client.api.sync$eval13084$fn__13105 invoke "sync.clj" 99]
[datomic.client.api.impl$eval1710$fn__1711$G__1701__1718 invoke "impl.clj" 28]
[datomic.client.api$q invokeStatic "api.clj" 288]
[datomic.client.api$q invoke "api.clj" 259]
[datomic.client.api$q invokeStatic "api.clj" 291]
[datomic.client.api$q doInvoke "api.clj" 259]
[clojure.lang.RestFn invoke "RestFn.java" 439]
[cloud_tester.core$fn__13655 invokeStatic "core.clj" 80]
[cloud_tester.core$fn__13655 invoke "core.clj" 76]
[clojure.core$binding_conveyor_fn$fn__5476 invoke "core.clj" 2022]
[clojure.lang.AFn call "AFn.java" 18]
[java.util.concurrent.FutureTask run "FutureTask.java" 266]
[java.util.concurrent.ThreadPoolExecutor runWorker "ThreadPoolExecutor.java" 1142]
[java.util.concurrent.ThreadPoolExecutor$Worker run "ThreadPoolExecutor.java" 617]
[java.lang.Thread run "Thread.java" 745]]}}]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment