SplitDataUnitPailStructure test with LZO compression

gistfile1.clj
Clojure
      
        
          
          
            REPL started; server listening on localhost port 16999
user=> (use 'cloudwh.split-pail)                                                                                                                              nil
user=> (in-ns 'cloudwh.split-pail)                                                                                                                            #<Namespace cloudwh.split-pail>         
cloudwh.split-pail=> (def pail5 (split-unit-tap "/tmp/data5"))
#'cloudwh.split-pail/pail5
cloudwh.split-pail=> pail5
#<PailTap PailTap["PailScheme[['pail_root', 'DataUnit']->[ALL]]"]["/tmp/data5"]"]>
cloudwh.split-pail=> (use 'cloudwh.load.exchange-rate.daily)
nil
cloudwh.split-pail=> thrift-daily-rates
[[#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:8, target_id:1), property:<ExchangeRatePropertyValue rate:0.6794274481581936>, creation_time:1335523591521)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:2, target_id:1), property:<ExchangeRatePropertyValue rate:0.8163999915122986>, creation_time:1335523591521)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:6, target_id:1), property:<ExchangeRatePropertyValue rate:0.10974150590753629>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:3, target_id:1), property:<ExchangeRatePropertyValue rate:0.6177828424312923>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:7, target_id:1), property:<ExchangeRatePropertyValue rate:0.10775423983225703>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:4, target_id:1), property:<ExchangeRatePropertyValue rate:0.1952175897262234>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:1, target_id:1), property:<ExchangeRatePropertyValue rate:1.0>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:5, target_id:1), property:<ExchangeRatePropertyValue rate:0.09197836348807117>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:8, target_id:2), property:<ExchangeRatePropertyValue rate:0.8322237325108527>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:2, target_id:2), property:<ExchangeRatePropertyValue rate:1.0>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:6, target_id:2), property:<ExchangeRatePropertyValue rate:0.13442124822202806>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:3, target_id:2), property:<ExchangeRatePropertyValue rate:0.7567158854165492>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:7, target_id:2), property:<ExchangeRatePropertyValue rate:0.13198706633087193>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:4, target_id:2), property:<ExchangeRatePropertyValue rate:0.23912002909823962>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:1, target_id:2), property:<ExchangeRatePropertyValue rate:1.2248897726561718>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:5, target_id:2), property:<ExchangeRatePropertyValue rate:0.11266335674219023>, creation_time:1335523591522)>>]]
cloudwh.split-pail=> (?- pail5 thrift-daily-rates)          
nil
cloudwh.split-pail=> (?- pail5 thrift-daily-rates)
IllegalArgumentException Pail already exists at path /tmp/data5 with spec {structure=com.photobox.pail.SplitDataUnitPailStructure, args={compressionCodec=lzo, compressionType=block}, format=SequenceFile}  backtype.hadoop.pail.Pail.create (Pail.java:153)
cloudwh.split-pail=> (to-pail "/tmp/data5" thrift-daily-rates)
nil
cloudwh.split-pail=> (to-pail "/tmp/data5" thrift-daily-rates)
nil
cloudwh.split-pail=> (to-pail "/tmp/data5" thrift-daily-rates)
nil
cloudwh.split-pail=> (def p5 (Pail. "/tmp/data5"))
#'cloudwh.split-pail/p5
cloudwh.split-pail=> (.consolidate p5)            
IOException Consolidator failed  backtype.hadoop.Consolidator.consolidate (Consolidator.java:110)

          
        

    
Please sign in to comment on this gist.
public lelayf / gist:2508394 Created 2012-04-27

public

lelayf / gist:2508394

Created 2012-04-27
