falcon.pl

# Cluster:
cluster {
  color 'local';
  description '';
  name 'local';
  
  interfaces (
        "hftp://localhost:41110" => { type => 'readonly', version => "0.20.2" },
        "hftp://localhost:41021" => { type => 'execute', version => "0.20.2" },
  );
  
  locations (
     staging => '/projects/falcon/staging',
     temp => '/tmp',
  );
}  

#Feed:
feed "in" => sub {
  description "in";

  partitions( { name => 'type' } );
  
  groups 'in';
  
  frequency '1h';
  
  timezone 'UTC';

  late_arrival cut_off => '6h';  

  clusters {
    cluster 'local', sub {
      validity "2013-01-01T00:00Z" => "2020-01-01T12:00Z";
      retention limit => '24h', action => 'delete';
    }
  };

  locations(
     '/data/in/${YEAR}/${MONTH}/${DAY}/${HOUR}' => 'data'
  );
  
  acl owner => "testuser", group => "group", permission => "0x644";
  
  schema "/schema/in/in.format.csv", provider => "csv"
}

#Process:
process 'wf-process' => sub {
  clusters {
    cluster 'local', sub {
      validity "2013-01-01T00:00Z" => "2013-01-01T02:00Z";
    };
  };

  parallel 1;
  order 'LIFO';
  frequency '1h';
  timezone 'UTC';
  
   inputs {
     input 'input', feed => 'in', start => 'now(0.0)', end => 'now(0.0)';
   };
   
   outputs {
     output 'output', feed => 'out', instance => 'today(0,0)';
   };
   
   workflow engine => 'oozie', path => '/app/mapred-wf.xml';
};