lelayf

(?<- (stdout) [?a-new]
     ([{:a 1 :b 2}
       {:a 1M :b 20}
       ] ?ds)
     (map ?ds [:a] :> ?a-new)
    ) 

lelayf

=> (?<- (stdout) [?a-new]
          ([{:a 1 :b 2}
            {:a 10 :b 20}] ?ds)
     (map ?ds [:a] :> ?a-new))

RESULTS
-----------------------
1
10

lelayf

# Get latest Sun Java SDK in v6 from Oracle 
wget http://download.oracle.com/otn-pub/java/jdk/6u31-b04/jdk-6u31-linux-x64-rpm.bin

# make it exec
chmod +x jdk-6u31-linux-x64-rpm.bin

# Install Java
sudo ./jdk-6u31-linux-x64-rpm.bin

# Check if the default java version is set to sun jdk

lelayf

(ns cloudwh.pail-test
  (:import [backtype.hadoop.pail Pail PailSpec SequenceFileFormat]))

(def pail3 (Pail/create "/tmp/data3"
                        (PailSpec. "SequenceFile"
                                   {SequenceFileFormat/TYPE_ARG SequenceFileFormat/TYPE_ARG_BLOCK
                                    SequenceFileFormat/CODEC_ARG SequenceFileFormat/CODEC_ARG_GZIP})))
(def tos (.openWrite pail3))

(.writeObject tos (byte-array [(byte 1) (byte 2) (byte 3)]))

lelayf

; Depends on my fork of dfs-datastores to include CODEC_ARG_LZO

(def pail4 (Pail/create "/tmp/data4" (PailSpec. "SequenceFile" {SequenceFileFormat/TYPE_ARG SequenceFileFormat/TYPE_ARG_BLOCK SequenceFileFormat/CODEC_ARG SequenceFileFormat/CODEC_ARG_LZO})))
; #'cloudwh.split-pail/pail4

(def tos (.openWrite pail4))
; #'cloudwh.split-pail/tos

; write this a couple times
(.writeObject tos (byte-array [(byte 3) (byte 4) (byte 5) (byte 122) (byte 12)]))

lelayf

REPL started; server listening on localhost port 16999
user=> (use 'cloudwh.split-pail)                                                                                                                              nil
user=> (in-ns 'cloudwh.split-pail)                                                                                                                            #<Namespace cloudwh.split-pail>         
cloudwh.split-pail=> (def pail5 (split-unit-tap "/tmp/data5"))
#'cloudwh.split-pail/pail5
cloudwh.split-pail=> pail5
#<PailTap PailTap["PailScheme[['pail_root', 'DataUnit']->[ALL]]"]["/tmp/data5"]"]>
cloudwh.split-pail=> (use 'cloudwh.load.exchange-rate.daily)
nil
cloudwh.split-pail=> thrift-daily-rates

lelayf

;; After renaming files in Pail to have .lzo extension
user=> (def indexer (com.hadoop.compression.lzo.LzoIndexer. (org.apache.hadoop.conf.Configuration.)))
log4j:WARN No appenders could be found for logger (org.apache.hadoop.conf.Configuration).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
#'user/indexer
user=> indexer
#<LzoIndexer com.hadoop.compression.lzo.LzoIndexer@381eb0c6>
user=> (.index indexer (org.apache.hadoop.fs.Path. "/tmp/data5"))                                    
nil

lelayf

;; thrifters

(defn thrift-order-item-line_price_net
  [id val]
  (DataUnit/order_item_property
    (OrderItemProperty.
      (OrderItemID/id id)
      (OrderItemPropertyValue/line_price_net val)
      (. (new Date) getTime))))

lelayf

# Compliler is jdk 1.7 but jre is 1.6
# I got error 
javac -source 1.6 -target 1.6 -bootclasspath /usr/lib/jvm/java-6-sun-1.6.0.26/jre/lib/rt.jar AvroDataFileReader.java 

lelayf

# save in $CAFFE_ROOT/examples/mnist 
../../python/classify.py --model_def ./lenet.prototxt --pretrained_model ./lenet_iter_10000 --mean_file='' --images_dim 28,28 --gpu --channel_swap '0' ~/lehedge/mnist-predict-100-twos.npy foo