lelayf / gist:2508394

Created April 27, 2012


    
    View gist:2508394 
  
  
      gistfile1.clj
    
    
        Raw
      
    
    File suppressed. Click to show.
  
  
            1
            2
            3
            4
            5
            6
            7
            8
            9
            10
            11
            12
            13
            14
            15
            16
            17
            18
            19
            20
            21
            22
            23
            24
            25
          
          
            REPL started; server listening on localhost port 16999
user=> (use 'cloudwh.split-pail)                                                                                                                              nil
user=> (in-ns 'cloudwh.split-pail)                                                                                                                            #<Namespace cloudwh.split-pail>         
cloudwh.split-pail=> (def pail5 (split-unit-tap "/tmp/data5"))
#'cloudwh.split-pail/pail5
cloudwh.split-pail=> pail5
#<PailTap PailTap["PailScheme[['pail_root', 'DataUnit']->[ALL]]"]["/tmp/data5"]"]>
cloudwh.split-pail=> (use 'cloudwh.load.exchange-rate.daily)
nil
cloudwh.split-pail=> thrift-daily-rates
[[#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:8, target_id:1), property:<ExchangeRatePropertyValue rate:0.6794274481581936>, creation_time:1335523591521)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:2, target_id:1), property:<ExchangeRatePropertyValue rate:0.8163999915122986>, creation_time:1335523591521)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:6, target_id:1), property:<ExchangeRatePropertyValue rate:0.10974150590753629>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:3, target_id:1), property:<ExchangeRatePropertyValue rate:0.6177828424312923>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:7, target_id:1), property:<ExchangeRatePropertyValue rate:0.10775423983225703>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:4, target_id:1), property:<ExchangeRatePropertyValue rate:0.1952175897262234>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:1, target_id:1), property:<ExchangeRatePropertyValue rate:1.0>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:5, target_id:1), property:<ExchangeRatePropertyValue rate:0.09197836348807117>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:8, target_id:2), property:<ExchangeRatePropertyValue rate:0.8322237325108527>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:2, target_id:2), property:<ExchangeRatePropertyValue rate:1.0>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:6, target_id:2), property:<ExchangeRatePropertyValue rate:0.13442124822202806>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:3, target_id:2), property:<ExchangeRatePropertyValue rate:0.7567158854165492>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:7, target_id:2), property:<ExchangeRatePropertyValue rate:0.13198706633087193>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:4, target_id:2), property:<ExchangeRatePropertyValue rate:0.23912002909823962>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:1, target_id:2), property:<ExchangeRatePropertyValue rate:1.2248897726561718>, creation_time:1335523591522)>>] [#<DataUnit <DataUnit fx_property:ExchangeRateProperty(id:ExchangeRateID(date:1335391200000, base_id:5, target_id:2), property:<ExchangeRatePropertyValue rate:0.11266335674219023>, creation_time:1335523591522)>>]]
cloudwh.split-pail=> (?- pail5 thrift-daily-rates)          
nil
cloudwh.split-pail=> (?- pail5 thrift-daily-rates)
IllegalArgumentException Pail already exists at path /tmp/data5 with spec {structure=com.photobox.pail.SplitDataUnitPailStructure, args={compressionCodec=lzo, compressionType=block}, format=SequenceFile}  backtype.hadoop.pail.Pail.create (Pail.java:153)
cloudwh.split-pail=> (to-pail "/tmp/data5" thrift-daily-rates)
nil
cloudwh.split-pail=> (to-pail "/tmp/data5" thrift-daily-rates)
nil
cloudwh.split-pail=> (to-pail "/tmp/data5" thrift-daily-rates)
nil
cloudwh.split-pail=> (def p5 (Pail. "/tmp/data5"))
#'cloudwh.split-pail/p5
cloudwh.split-pail=> (.consolidate p5)            
IOException Consolidator failed  backtype.hadoop.Consolidator.consolidate (Consolidator.java:110)