Skip to content

Instantly share code, notes, and snippets.

@jvwing
Created September 19, 2016 21:53
Show Gist options
  • Save jvwing/45c17685770a096895f3d88845bac132 to your computer and use it in GitHub Desktop.
Save jvwing/45c17685770a096895f3d88845bac132 to your computer and use it in GitHub Desktop.
NiFi template demonstrating use of built-in fields in Expression Language.
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><template><description>Tests various built-in FlowFile fields using Expression Language.</description><name>FlowFileFieldsTest</name><snippet><connections><id>ffa84c5c-a236-45f6-a139-51d826e73d3c</id><parentGroupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</groupId><id>6f7d661c-e476-48b7-a6fe-e06ea4dc4fd3</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</groupId><id>53074840-f32c-459f-8739-7ed866bf7c3f</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>2c2150f3-b67f-4bbe-ad88-97e4adfc7dd0</id><parentGroupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</groupId><id>53074840-f32c-459f-8739-7ed866bf7c3f</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</groupId><id>a9796d42-e386-4905-8539-00422edd5f9e</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><processors><id>a9796d42-e386-4905-8539-00422edd5f9e</id><parentGroupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</parentGroupId><position><x>122.0</x><y>-5.0</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>File Size</key><value><description>The size of the file that will be used</description><displayName>File Size</displayName><dynamic>false</dynamic><name>File Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Batch Size</key><value><defaultValue>1</defaultValue><description>The number of FlowFiles to be transferred in each invocation</description><displayName>Batch Size</displayName><dynamic>false</dynamic><name>Batch Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Data Format</key><value><allowableValues><displayName>Binary</displayName><value>Binary</value></allowableValues><allowableValues><displayName>Text</displayName><value>Text</value></allowableValues><defaultValue>Binary</defaultValue><description>Specifies whether the data should be Text or Binary</description><displayName>Data Format</displayName><dynamic>false</dynamic><name>Data Format</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Unique FlowFiles</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput</description><displayName>Unique FlowFiles</displayName><dynamic>false</dynamic><name>Unique FlowFiles</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>File Size</key><value>13023 B</value></entry><entry><key>Batch Size</key><value>1</value></entry><entry><key>Data Format</key><value>Binary</value></entry><entry><key>Unique FlowFiles</key><value>false</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>10 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>GenerateFlowFile</name><relationships><autoTerminate>false</autoTerminate><description></description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.GenerateFlowFile</type></processors><processors><id>53074840-f32c-459f-8739-7ed866bf7c3f</id><parentGroupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</parentGroupId><position><x>123.0</x><y>213.0</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Delete Attributes Expression</key><value><description>Regular expression for attributes to be deleted from flowfiles.</description><displayName>Delete Attributes Expression</displayName><dynamic>false</dynamic><name>Delete Attributes Expression</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.absolute.path</key><value><description></description><displayName>test.absolute.path</displayName><dynamic>true</dynamic><name>test.absolute.path</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.alternate.identifier</key><value><description></description><displayName>test.alternate.identifier</displayName><dynamic>true</dynamic><name>test.alternate.identifier</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.discard.reason</key><value><description></description><displayName>test.discard.reason</displayName><dynamic>true</dynamic><name>test.discard.reason</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.entryDate</key><value><description></description><displayName>test.entryDate</displayName><dynamic>true</dynamic><name>test.entryDate</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.filename</key><value><description></description><displayName>test.filename</displayName><dynamic>true</dynamic><name>test.filename</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.fileSize</key><value><description></description><displayName>test.fileSize</displayName><dynamic>true</dynamic><name>test.fileSize</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.fileSizeKB</key><value><description></description><displayName>test.fileSizeKB</displayName><dynamic>true</dynamic><name>test.fileSizeKB</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.flowFileId</key><value><description></description><displayName>test.flowFileId</displayName><dynamic>true</dynamic><name>test.flowFileId</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.lastQueueDate</key><value><description></description><displayName>test.lastQueueDate</displayName><dynamic>true</dynamic><name>test.lastQueueDate</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.lineageStartDate</key><value><description></description><displayName>test.lineageStartDate</displayName><dynamic>true</dynamic><name>test.lineageStartDate</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.lineageStartDateFormat</key><value><description></description><displayName>test.lineageStartDateFormat</displayName><dynamic>true</dynamic><name>test.lineageStartDateFormat</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.mime.type</key><value><description></description><displayName>test.mime.type</displayName><dynamic>true</dynamic><name>test.mime.type</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.path</key><value><description></description><displayName>test.path</displayName><dynamic>true</dynamic><name>test.path</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.priority</key><value><description></description><displayName>test.priority</displayName><dynamic>true</dynamic><name>test.priority</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.queueDateIndex</key><value><description></description><displayName>test.queueDateIndex</displayName><dynamic>true</dynamic><name>test.queueDateIndex</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.size</key><value><description></description><displayName>test.size</displayName><dynamic>true</dynamic><name>test.size</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>test.uuid</key><value><description></description><displayName>test.uuid</displayName><dynamic>true</dynamic><name>test.uuid</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Delete Attributes Expression</key></entry><entry><key>test.absolute.path</key><value>${absolute.path}</value></entry><entry><key>test.alternate.identifier</key><value>${alternate.identifier}</value></entry><entry><key>test.discard.reason</key><value>${discard.reason}</value></entry><entry><key>test.entryDate</key><value>${entryDate}</value></entry><entry><key>test.filename</key><value>${filename}</value></entry><entry><key>test.fileSize</key><value>${fileSize}</value></entry><entry><key>test.fileSizeKB</key><value>${fileSize:toNumber():divide(1024)}</value></entry><entry><key>test.flowFileId</key><value>${flowFileId}</value></entry><entry><key>test.lastQueueDate</key><value>${lastQueueDate}</value></entry><entry><key>test.lineageStartDate</key><value>${lineageStartDate}</value></entry><entry><key>test.lineageStartDateFormat</key><value>${now():toNumber():minus(${lineageStartDate}):format(&quot;HH:mm:ss&quot;)}</value></entry><entry><key>test.mime.type</key><value>${mime.type}</value></entry><entry><key>test.path</key><value>${path}</value></entry><entry><key>test.priority</key><value>${priority}</value></entry><entry><key>test.queueDateIndex</key><value>${queueDateIndex}</value></entry><entry><key>test.size</key><value>${size}</value></entry><entry><key>test.uuid</key><value>${uuid}</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>Evaluate Attributes</name><relationships><autoTerminate>false</autoTerminate><description>All FlowFiles are routed to this relationship</description><name>success</name></relationships><state>RUNNING</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.attributes.UpdateAttribute</type></processors><processors><id>6f7d661c-e476-48b7-a6fe-e06ea4dc4fd3</id><parentGroupId>8524b3ae-950f-49c5-a936-44dd56b5a4d1</parentGroupId><position><x>123.0</x><y>423.0</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Log Level</key><value><allowableValues><displayName>trace</displayName><value>trace</value></allowableValues><allowableValues><displayName>debug</displayName><value>debug</value></allowableValues><allowableValues><displayName>info</displayName><value>info</value></allowableValues><allowableValues><displayName>warn</displayName><value>warn</value></allowableValues><allowableValues><displayName>error</displayName><value>error</value></allowableValues><defaultValue>info</defaultValue><description>The Log Level to use when logging the Attributes</description><displayName>Log Level</displayName><dynamic>false</dynamic><name>Log Level</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Log Payload</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the FlowFile's payload will be logged, in addition to its attributes; otherwise, just the Attributes will be logged.</description><displayName>Log Payload</displayName><dynamic>false</dynamic><name>Log Payload</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Attributes to Log</key><value><description>A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.</description><displayName>Attributes to Log</displayName><dynamic>false</dynamic><name>Attributes to Log</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Attributes to Ignore</key><value><description>A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.</description><displayName>Attributes to Ignore</displayName><dynamic>false</dynamic><name>Attributes to Ignore</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Log prefix</key><value><description>Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.</description><displayName>Log prefix</displayName><dynamic>false</dynamic><name>Log prefix</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Log Level</key><value>info</value></entry><entry><key>Log Payload</key><value>false</value></entry><entry><key>Attributes to Log</key></entry><entry><key>Attributes to Ignore</key></entry><entry><key>Log prefix</key></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>LogAttribute</name><relationships><autoTerminate>true</autoTerminate><description>All FlowFiles are routed to this relationship</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.LogAttribute</type></processors></snippet><timestamp>09/19/2016 21:51:36 UTC</timestamp></template>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment