public
Last active

TemplateTap + thrift + Hfs-seqfile / hfs-lzo-thrift

  • Download Gist
gistfile1.clj
Clojure
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
;; thrifters
 
(defn thrift-order-item-line_price_net
[id val]
(DataUnit/order_item_property
(OrderItemProperty.
(OrderItemID/id id)
(OrderItemPropertyValue/line_price_net val)
(. (new Date) getTime))))
 
(defn thrift-order-item-line_price_inc
[id val]
(DataUnit/order_item_property
(OrderItemProperty.
(OrderItemID/id id)
(OrderItemPropertyValue/line_price_inc val)
(. (new Date) getTime))))
 
;; convert one record of the form id, field-value1, field-value2
;; to 2 records field-name1, thrift-value1 and field-name2, thrift-value2
;; this is done to facilitate templating in sink tap based on fields names
 
(defmapcatop thriftcat [id p1 p2]
[["line_price_inc" (thrift-order-item-line_price_inc id p1)]
["line_price_net" (thrift-order-item-line_price_net id p2)]])
 
;; fetch MySQL data from source-tap (dbmigrate) and dump to TemplateTap
;; one file is generated per fetched field
;; so with one source query of N fields we can generate N files
 
;; Using hfs-seqfile : works fine
 
(defn dump-order-item-prices []
(let [fields ["id" "line_price_inc" "line_price_net"]
cvars (vec (map #(str "?" % "") fields))]
(?<- (hfs-seqfile "/data/OrderItemProperty" :sinkmode :replace
:sink-template "%s/"
:templatefields ["?prop"]
:outfields ["?thrift-object"])
[?prop ?thrift-object]
((source-tap fields) :>> cvars )
(thriftcat :<< cvars :> ?prop ?thrift-object )
(:distinct false))))
 
;; Using hfs-lzo-thrift : it breaks
 
(defn dump-order-item-prices []
(let [fields ["id" "line_price_inc" "line_price_net"]
cvars (vec (map #(str "?" % "") fields))]
(?<- (hfs-lzo-thrift "/data/OrderItemProperty" DataUnit
:sinkmode :replace
:sink-template "%s/"
:templatefields ["?prop"]
:outfields ["?thrift-object"])
[?prop ?thrift-object]
((source-tap fields) :>> cvars )
(thriftcat :<< cvars :> ?prop ?thrift-object )
(:distinct false))))

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.