Created
December 12, 2016 14:35
-
-
Save mattyb149/c2890dd17ad1ba21199718f6eea88279 to your computer and use it in GitHub Desktop.
NiFi template to check for the existence of a file, then transfers all flow files to success
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0" ?> | |
<template encoding-version="1.0"> | |
<description>This template uses ExecuteScript to check for the existence of a file, then transfers all flow files to success while the file exists. This can be used to start and stop the movement of files external to NiFi.</description> | |
<groupId>f35864f5-0158-1000-f261-0c18ddb7fb1b</groupId> | |
<name>FetchOnFileExists</name> | |
<snippet> | |
<connections> | |
<id>f3597301-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>f35864f5-0158-1000-0000-000000000000</groupId> | |
<id>f3590d6b-0158-1000-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>f35864f5-0158-1000-0000-000000000000</groupId> | |
<id>f358fd6b-0158-1000-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>f359ac50-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>f35864f5-0158-1000-0000-000000000000</groupId> | |
<id>f3591ea9-0158-1000-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>f35864f5-0158-1000-0000-000000000000</groupId> | |
<id>f3590d6b-0158-1000-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>f359ce3d-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>f35864f5-0158-1000-0000-000000000000</groupId> | |
<id>f3595a53-0158-1000-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>f35864f5-0158-1000-0000-000000000000</groupId> | |
<id>f3591ea9-0158-1000-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<processors> | |
<id>f358fd6b-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.0</x> | |
<y>0.0</y> | |
</position> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Input Directory</key> | |
<value> | |
<name>Input Directory</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Recurse Subdirectories</key> | |
<value> | |
<name>Recurse Subdirectories</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Input Directory Location</key> | |
<value> | |
<name>Input Directory Location</name> | |
</value> | |
</entry> | |
<entry> | |
<key>File Filter</key> | |
<value> | |
<name>File Filter</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Path Filter</key> | |
<value> | |
<name>Path Filter</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Minimum File Age</key> | |
<value> | |
<name>Minimum File Age</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Maximum File Age</key> | |
<value> | |
<name>Maximum File Age</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Minimum File Size</key> | |
<value> | |
<name>Minimum File Size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Maximum File Size</key> | |
<value> | |
<name>Maximum File Size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Ignore Hidden Files</key> | |
<value> | |
<name>Ignore Hidden Files</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Input Directory</key> | |
<value>/Users/mburgess/tmp/wait</value> | |
</entry> | |
<entry> | |
<key>Recurse Subdirectories</key> | |
<value>true</value> | |
</entry> | |
<entry> | |
<key>Input Directory Location</key> | |
<value>Local</value> | |
</entry> | |
<entry> | |
<key>File Filter</key> | |
<value>^(?!_success$).+</value> | |
</entry> | |
<entry> | |
<key>Path Filter</key> | |
</entry> | |
<entry> | |
<key>Minimum File Age</key> | |
<value>0 sec</value> | |
</entry> | |
<entry> | |
<key>Maximum File Age</key> | |
</entry> | |
<entry> | |
<key>Minimum File Size</key> | |
<value>0 B</value> | |
</entry> | |
<entry> | |
<key>Maximum File Size</key> | |
</entry> | |
<entry> | |
<key>Ignore Hidden Files</key> | |
<value>true</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>ListFile</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.standard.ListFile</type> | |
</processors> | |
<processors> | |
<id>f3590d6b-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.0</x> | |
<y>228.0</y> | |
</position> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Script Engine</key> | |
<value> | |
<name>Script Engine</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Script File</key> | |
<value> | |
<name>Script File</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Script Body</key> | |
<value> | |
<name>Script Body</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Module Directory</key> | |
<value> | |
<name>Module Directory</name> | |
</value> | |
</entry> | |
<entry> | |
<key>batchSize</key> | |
<value> | |
<name>batchSize</name> | |
</value> | |
</entry> | |
<entry> | |
<key>fileExists</key> | |
<value> | |
<name>fileExists</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Script Engine</key> | |
<value>Groovy</value> | |
</entry> | |
<entry> | |
<key>Script File</key> | |
</entry> | |
<entry> | |
<key>Script Body</key> | |
<value>if(!new File(fileExists.value).exists()) return | |
session.transfer(session.get(batchSize.asInteger()), REL_SUCCESS)</value> | |
</entry> | |
<entry> | |
<key>Module Directory</key> | |
</entry> | |
<entry> | |
<key>batchSize</key> | |
<value>1000</value> | |
</entry> | |
<entry> | |
<key>fileExists</key> | |
<value>/Users/mburgess/tmp/wait/_success</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>ExecuteScript</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.script.ExecuteScript</type> | |
</processors> | |
<processors> | |
<id>f3591ea9-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.0</x> | |
<y>457.0</y> | |
</position> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>File to Fetch</key> | |
<value> | |
<name>File to Fetch</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Completion Strategy</key> | |
<value> | |
<name>Completion Strategy</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Move Destination Directory</key> | |
<value> | |
<name>Move Destination Directory</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Move Conflict Strategy</key> | |
<value> | |
<name>Move Conflict Strategy</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Log level when file not found</key> | |
<value> | |
<name>Log level when file not found</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Log level when permission denied</key> | |
<value> | |
<name>Log level when permission denied</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>File to Fetch</key> | |
<value>${absolute.path}/${filename}</value> | |
</entry> | |
<entry> | |
<key>Completion Strategy</key> | |
<value>Move File</value> | |
</entry> | |
<entry> | |
<key>Move Destination Directory</key> | |
<value>/Users/mburgess/tmp/done</value> | |
</entry> | |
<entry> | |
<key>Move Conflict Strategy</key> | |
<value>Fail</value> | |
</entry> | |
<entry> | |
<key>Log level when file not found</key> | |
<value>ERROR</value> | |
</entry> | |
<entry> | |
<key>Log level when permission denied</key> | |
<value>ERROR</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>FetchFile</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>not.found</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>permission.denied</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.standard.FetchFile</type> | |
</processors> | |
<processors> | |
<id>f3595a53-0158-1000-0000-000000000000</id> | |
<parentGroupId>f35864f5-0158-1000-0000-000000000000</parentGroupId> | |
<position> | |
<x>2.0</x> | |
<y>673.0</y> | |
</position> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Log Level</key> | |
<value> | |
<name>Log Level</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Log Payload</key> | |
<value> | |
<name>Log Payload</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Attributes to Log</key> | |
<value> | |
<name>Attributes to Log</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Attributes to Ignore</key> | |
<value> | |
<name>Attributes to Ignore</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Log prefix</key> | |
<value> | |
<name>Log prefix</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Log Level</key> | |
<value>info</value> | |
</entry> | |
<entry> | |
<key>Log Payload</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>Attributes to Log</key> | |
</entry> | |
<entry> | |
<key>Attributes to Ignore</key> | |
</entry> | |
<entry> | |
<key>Log prefix</key> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>LogAttribute</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.standard.LogAttribute</type> | |
</processors> | |
</snippet> | |
<timestamp>12/12/2016 09:31:22 EST</timestamp> | |
</template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment