Skip to content

Instantly share code, notes, and snippets.

@ijokarumawak

ijokarumawak/ExecuteStreamCommandExample.xml

Last active February 15, 2022 11:20
Show Gist options
  • Save ijokarumawak/64c8d8a14e7e280249c239074f433b13 to your computer and use it in GitHub Desktop.
Save ijokarumawak/64c8d8a14e7e280249c239074f433b13 to your computer and use it in GitHub Desktop.
Download ZIP
NiFi Template Files
Raw
_NiFi_template_files.md

NiFi Template Files

  • ExecuteStreamCommandExample : GenerateFlowFileで生成したtextデータをExecuteStreamCommandでwcコマンドに渡し、wcの結果をLogAttributeで出力します。
Raw
ExecuteStreamCommandExample.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template>
<description>GenerateFlowFileで生成したtextデータをExecuteStreamCommandでwcコマンドに渡し、wcの結果をLogAttributeで出力します。</description>
<name>ExecuteStreamCommandExample</name>
<snippet>
<connections>
<id>6458677b-cfba-442c-8d6f-c90a8f0c67ee</id>
<parentGroupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</parentGroupId>
<backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>0</backPressureObjectThreshold>
<destination>
<groupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</groupId>
<id>2fe39650-a0d2-4391-bcf0-87247c867c97</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>output stream</selectedRelationships>
<source>
<groupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</groupId>
<id>ac4d9845-8099-4827-87cb-9ec0d3c2a0ea</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>174b1a53-dfcc-43a2-96b5-ca706dad2814</id>
<parentGroupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</parentGroupId>
<backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>0</backPressureObjectThreshold>
<destination>
<groupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</groupId>
<id>ac4d9845-8099-4827-87cb-9ec0d3c2a0ea</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</groupId>
<id>7eaca58a-fb6b-4e18-a072-8f95ce5d62cd</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<processors>
<id>7eaca58a-fb6b-4e18-a072-8f95ce5d62cd</id>
<parentGroupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</parentGroupId>
<position>
<x>204.0</x>
<y>14.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<defaultConcurrentTasks>
<entry>
<key>TIMER_DRIVEN</key>
<value>1</value>
</entry>
<entry>
<key>EVENT_DRIVEN</key>
<value>0</value>
</entry>
<entry>
<key>CRON_DRIVEN</key>
<value>1</value>
</entry>
</defaultConcurrentTasks>
<defaultSchedulingPeriod>
<entry>
<key>TIMER_DRIVEN</key>
<value>0 sec</value>
</entry>
<entry>
<key>CRON_DRIVEN</key>
<value>* * * * * ?</value>
</entry>
</defaultSchedulingPeriod>
<descriptors>
<entry>
<key>File Size</key>
<value>
<description>The size of the file that will be used</description>
<displayName>File Size</displayName>
<dynamic>false</dynamic>
<name>File Size</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Batch Size</key>
<value>
<defaultValue>1</defaultValue>
<description>The number of FlowFiles to be transferred in each invocation</description>
<displayName>Batch Size</displayName>
<dynamic>false</dynamic>
<name>Batch Size</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Data Format</key>
<value>
<allowableValues>
<displayName>Binary</displayName>
<value>Binary</value>
</allowableValues>
<allowableValues>
<displayName>Text</displayName>
<value>Text</value>
</allowableValues>
<defaultValue>Binary</defaultValue>
<description>Specifies whether the data should be Text or Binary</description>
<displayName>Data Format</displayName>
<dynamic>false</dynamic>
<name>Data Format</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>
<allowableValues>
<displayName>true</displayName>
<value>true</value>
</allowableValues>
<allowableValues>
<displayName>false</displayName>
<value>false</value>
</allowableValues>
<defaultValue>false</defaultValue>
<description>If true, each FlowFile that is generated will be unique. If false, a random
value will be generated and all FlowFiles will get the same content but this offers much
higher throughput
</description>
<displayName>Unique FlowFiles</displayName>
<dynamic>false</dynamic>
<name>Unique FlowFiles</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>File Size</key>
<value>10b</value>
</entry>
<entry>
<key>Batch Size</key>
</entry>
<entry>
<key>Data Format</key>
<value>Text</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>5 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>GenerateFlowFile</name>
<relationships>
<autoTerminate>false</autoTerminate>
<description></description>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<supportsEventDriven>false</supportsEventDriven>
<supportsParallelProcessing>true</supportsParallelProcessing>
<type>org.apache.nifi.processors.standard.GenerateFlowFile</type>
</processors>
<processors>
<id>2fe39650-a0d2-4391-bcf0-87247c867c97</id>
<parentGroupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</parentGroupId>
<position>
<x>206.0</x>
<y>391.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<defaultConcurrentTasks>
<entry>
<key>TIMER_DRIVEN</key>
<value>1</value>
</entry>
<entry>
<key>EVENT_DRIVEN</key>
<value>0</value>
</entry>
<entry>
<key>CRON_DRIVEN</key>
<value>1</value>
</entry>
</defaultConcurrentTasks>
<defaultSchedulingPeriod>
<entry>
<key>TIMER_DRIVEN</key>
<value>0 sec</value>
</entry>
<entry>
<key>CRON_DRIVEN</key>
<value>* * * * * ?</value>
</entry>
</defaultSchedulingPeriod>
<descriptors>
<entry>
<key>Log Level</key>
<value>
<allowableValues>
<displayName>trace</displayName>
<value>trace</value>
</allowableValues>
<allowableValues>
<displayName>debug</displayName>
<value>debug</value>
</allowableValues>
<allowableValues>
<displayName>info</displayName>
<value>info</value>
</allowableValues>
<allowableValues>
<displayName>warn</displayName>
<value>warn</value>
</allowableValues>
<allowableValues>
<displayName>error</displayName>
<value>error</value>
</allowableValues>
<defaultValue>info</defaultValue>
<description>The Log Level to use when logging the Attributes</description>
<displayName>Log Level</displayName>
<dynamic>false</dynamic>
<name>Log Level</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Log Payload</key>
<value>
<allowableValues>
<displayName>true</displayName>
<value>true</value>
</allowableValues>
<allowableValues>
<displayName>false</displayName>
<value>false</value>
</allowableValues>
<defaultValue>false</defaultValue>
<description>If true, the FlowFile's payload will be logged, in addition to its attributes;
otherwise, just the Attributes will be logged.
</description>
<displayName>Log Payload</displayName>
<dynamic>false</dynamic>
<name>Log Payload</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Attributes to Log</key>
<value>
<description>A comma-separated list of Attributes to Log. If not specified, all attributes
will be logged.
</description>
<displayName>Attributes to Log</displayName>
<dynamic>false</dynamic>
<name>Attributes to Log</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Attributes to Ignore</key>
<value>
<description>A comma-separated list of Attributes to ignore. If not specified, no attributes
will be ignored.
</description>
<displayName>Attributes to Ignore</displayName>
<dynamic>false</dynamic>
<name>Attributes to Ignore</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Log prefix</key>
<value>
<description>Log prefix appended to the log lines. It helps to distinguish the output of
multiple LogAttribute processors.
</description>
<displayName>Log prefix</displayName>
<dynamic>false</dynamic>
<name>Log prefix</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>true</supportsEl>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Log Level</key>
</entry>
<entry>
<key>Log Payload</key>
<value>true</value>
</entry>
<entry>
<key>Attributes to Log</key>
</entry>
<entry>
<key>Attributes to Ignore</key>
</entry>
<entry>
<key>Log prefix</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>LogAttribute</name>
<relationships>
<autoTerminate>true</autoTerminate>
<description>All FlowFiles are routed to this relationship</description>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<supportsEventDriven>true</supportsEventDriven>
<supportsParallelProcessing>true</supportsParallelProcessing>
<type>org.apache.nifi.processors.standard.LogAttribute</type>
</processors>
<processors>
<id>ac4d9845-8099-4827-87cb-9ec0d3c2a0ea</id>
<parentGroupId>7d6ffdcd-6518-4bff-9b25-31ae93f65c01</parentGroupId>
<position>
<x>204.0</x>
<y>201.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<defaultConcurrentTasks>
<entry>
<key>TIMER_DRIVEN</key>
<value>1</value>
</entry>
<entry>
<key>EVENT_DRIVEN</key>
<value>0</value>
</entry>
<entry>
<key>CRON_DRIVEN</key>
<value>1</value>
</entry>
</defaultConcurrentTasks>
<defaultSchedulingPeriod>
<entry>
<key>TIMER_DRIVEN</key>
<value>0 sec</value>
</entry>
<entry>
<key>CRON_DRIVEN</key>
<value>* * * * * ?</value>
</entry>
</defaultSchedulingPeriod>
<descriptors>
<entry>
<key>Command Arguments</key>
<value>
<description>The arguments to supply to the executable delimited by the ';' character.
</description>
<displayName>Command Arguments</displayName>
<dynamic>false</dynamic>
<name>Command Arguments</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>true</supportsEl>
</value>
</entry>
<entry>
<key>Command Path</key>
<value>
<description>Specifies the command to be executed; if just the name of an executable is
provided, it must be in the user's environment PATH.
</description>
<displayName>Command Path</displayName>
<dynamic>false</dynamic>
<name>Command Path</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>true</supportsEl>
</value>
</entry>
<entry>
<key>Ignore STDIN</key>
<value>
<allowableValues>
<displayName>true</displayName>
<value>true</value>
</allowableValues>
<allowableValues>
<displayName>false</displayName>
<value>false</value>
</allowableValues>
<defaultValue>false</defaultValue>
<description>If true, the contents of the incoming flowfile will not be passed to the
executing command
</description>
<displayName>Ignore STDIN</displayName>
<dynamic>false</dynamic>
<name>Ignore STDIN</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Working Directory</key>
<value>
<description>The directory to use as the current working directory when executing the
command
</description>
<displayName>Working Directory</displayName>
<dynamic>false</dynamic>
<name>Working Directory</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>true</supportsEl>
</value>
</entry>
<entry>
<key>Argument Delimiter</key>
<value>
<defaultValue>;</defaultValue>
<description>Delimiter to use to separate arguments for a command [default: ;]. Must be a
single character
</description>
<displayName>Argument Delimiter</displayName>
<dynamic>false</dynamic>
<name>Argument Delimiter</name>
<required>true</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Output Destination Attribute</key>
<value>
<description>If set, the output of the stream command will be put into an attribute of the
original FlowFile instead of a separate FlowFile. There will no longer be a relationship
for 'output stream'. The value of this property will be the key for the output
attribute.
</description>
<displayName>Output Destination Attribute</displayName>
<dynamic>false</dynamic>
<name>Output Destination Attribute</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
<entry>
<key>Max Attribute Length</key>
<value>
<defaultValue>256</defaultValue>
<description>If routing the output of the stream command to an attribute, the number of
characters put to the attribute value will be at most this amount. This is important
because attributes are held in memory and large attributes will quickly cause out of
memory issues. If the output goes longer than this value, it will truncated to fit.
Consider making this smaller if able.
</description>
<displayName>Max Attribute Length</displayName>
<dynamic>false</dynamic>
<name>Max Attribute Length</name>
<required>false</required>
<sensitive>false</sensitive>
<supportsEl>false</supportsEl>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Command Arguments</key>
</entry>
<entry>
<key>Command Path</key>
<value>wc</value>
</entry>
<entry>
<key>Ignore STDIN</key>
</entry>
<entry>
<key>Working Directory</key>
</entry>
<entry>
<key>Argument Delimiter</key>
</entry>
<entry>
<key>Output Destination Attribute</key>
</entry>
<entry>
<key>Max Attribute Length</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ExecuteStreamCommand</name>
<relationships>
<autoTerminate>true</autoTerminate>
<description>FlowFiles that were successfully processed</description>
<name>original</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<description>The destination path for the flow file created from the command's output</description>
<name>output stream</name>
</relationships>
<state>STOPPED</state>
<style/>
<supportsEventDriven>true</supportsEventDriven>
<supportsParallelProcessing>true</supportsParallelProcessing>
<type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type>
</processors>
</snippet>
<timestamp>05/16/2016 23:26:28 UTC</timestamp>
</template>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment