Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
(use 'forma.hadoop.jobs.scatter)
;; put stuff in pail
(let [pail-path "/tmp/newpail"
vcf-src [(thrift/DataChunk* "vcf"
(thrift/ModisPixelLocation* "500" 28 8 0 0)
25
"00")]
gadm-src [(thrift/DataChunk* "gadm"
(thrift/ModisPixelLocation* "500" 28 8 0 0)
16435
"00")]
vcf-query (<- [?dc]
(vcf-src ?dc))
gadm-query (<- [?dc]
(gadm-src ?dc))]
;; Data appear in pail no problem:
(to-pail pail-path vcf-query)
(to-pail pail-path gadm-query))
;; helpers
(defn unpack-pixel-loc
[loc]
(let [p-loc (.getPixelLocation (:property (bean loc)))]
[(.getResolution p-loc)
(.getTileH p-loc)
(.getTileV p-loc)
(.getSample p-loc)
(.getLine p-loc)]))
(defn unpack-long
"Unpack long value from static source"
[dv]
(.getLongVal dv))
;; get values out of pail
;; data comes out when there's one source, but joining on sres h v s l produces nothing
;; these queries run just fine - we get the expected values back
(let [pail-path "/tmp/newpail"
vcf-src (split-chunk-tap pail-path ["vcf"])]
(??<- [?sres ?h ?v ?s ?l ?vcf]
(vcf-src _ ?vcf-obj)
(bean ?vcf-obj :> ?vcf-map)
(get ?vcf-map :locationProperty :> ?vcf-loc)
(unpack-pixel-loc ?vcf-loc :> ?sres ?h ?v ?s ?l)
(get ?vcf-map :chunkValue :> ?vcf-dv)
(unpack-long ?vcf-dv :> ?vcf)))
=> (["500" 28 8 0 0 25])
(let [pail-path "/tmp/newpail"
gadm-src (split-chunk-tap pail-path ["gadm"])]
(??<- [?sres ?h ?v ?s ?l ?gadm]
(gadm-src _ ?gadm-obj)
(bean ?gadm-obj :> ?gadm-map)
(get ?gadm-map :locationProperty :> ?gadm-loc)
(unpack-pixel-loc ?gadm-loc :> ?sres ?h ?v ?s ?l)
(get ?gadm-map :chunkValue :> ?gadm-dv)
(unpack-long ?gadm-dv :> ?gadm)))
=> (["500" 28 8 0 0 16435])
;; runs but no output:
(let [pail-path "/tmp/newpail"
gadm-src (split-chunk-tap pail-path ["gadm"])
vcf-src (split-chunk-tap pail-path ["vcf"])]
(??<- [?sres ?h ?v ?s ?l ?gadm ?vcf]
(gadm-src _ ?gadm-obj)
(bean ?gadm-obj :> ?gadm-map)
(get ?gadm-map :locationProperty :> ?gadm-loc)
(unpack-pixel-loc ?gadm-loc :> ?sres ?h ?v ?s ?l)
(get ?gadm-map :chunkValue :> ?gadm-dv)
(unpack-long ?gadm-dv :> ?gadm)
(vcf-src _ ?vcf-obj)
(bean ?vcf-obj :> ?vcf-map)
(get ?vcf-map :locationProperty :> ?vcf-loc)
(unpack-pixel-loc ?vcf-loc :> ?sres ?h ?v ?s ?l)
(get ?vcf-map :chunkValue :> ?vcf-dv)
(unpack-long ?vcf-dv :> ?vcf)))
=> ()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment