BenjyO238

atlas_poc

BenjyO238

;;Gist start:
;; Not all supporting functions are returned. Just the main ones used to do the multi-threaded query
;; to hbase. I found a year of data returned in about 2 seconds vs. 20 or so seconds for similar 
;; query on a single thread. 

;; pids:
(def active-pids (slurp "active_pids_may1_13.csv")) ;; loaded pids from oracle
(def pid-pool (drop 1 (into [] (string/split active-pids #"\n")))) 
(def pid-to-get (Integer/parseInt (nth pid-pool 10000)))  ;; get random pid

BenjyO238

(defn get-pids-for-day [fut-func f pp]  ; add date rangepp
  "fut-func is apply-query-to-pid, f is p-day-qry, pp partitioned-pids.
   Iterate through each n pids making futures for n pids (batch size)"
  (let [results []] ;vec inside vec to mirror readings/days 
    (for [pid-pool pp]
      (let [fut (fut-func pid-pool f)]      
        (while (not (every? future-done? fut))
          (Thread/sleep 500)
          (prn (str "waiting for current future to finish..." (first pid-pool))))   
         (conj results (apply combine-readings (map #(identity @ %) fut)))))))