Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
NiFi, A workaround if sending large data via S2S does not work due to taking more than the default 'idle connection expiration' (30s)
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.2">
<description></description>
<groupId>aa0b8252-0162-1000-87e0-8c2881d45fa1</groupId>
<name>Large S2S Workaround</name>
<snippet>
<processGroups>
<id>59d79054-2944-3b82-0000-000000000000</id>
<parentGroupId>0329e4dc-ff21-384f-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<comments></comments>
<contents>
<connections>
<id>81d2128b-ae34-3105-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>8170b2d5-b073-361a-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>segments</selectedRelationships>
<source>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>44ea26dc-eb7b-367b-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>abd350ed-4b4c-3aeb-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>44ea26dc-eb7b-367b-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>ec5cb9c9-8b28-3568-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>b1980502-9d12-32fb-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>edb5bf2c-1b6b-317b-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>8170b2d5-b073-361a-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>7f412446-dc22-313e-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>ec5cb9c9-8b28-3568-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>59d79054-2944-3b82-0000-000000000000</groupId>
<id>47ee19c9-8807-3f55-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<labels>
<id>d7cf9dc5-f685-30e7-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<position>
<x>1943.0</x>
<y>251.0</y>
</position>
<height>53.0</height>
<label>Segments created from the same FlowFile
will have the same target.node attribute.</label>
<style>
<entry>
<key>font-size</key>
<value>12px</value>
</entry>
</style>
<width>267.0</width>
</labels>
<processors>
<id>8170b2d5-b073-361a-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<position>
<x>1543.0</x>
<y>204.0</y>
</position>
<bundle>
<artifact>nifi-update-attribute-nar</artifact>
<group>org.apache.nifi</group>
<version>1.7.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Delete Attributes Expression</key>
<value>
<name>Delete Attributes Expression</name>
</value>
</entry>
<entry>
<key>Store State</key>
<value>
<name>Store State</name>
</value>
</entry>
<entry>
<key>Stateful Variables Initial Value</key>
<value>
<name>Stateful Variables Initial Value</name>
</value>
</entry>
<entry>
<key>target.node</key>
<value>
<name>target.node</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Delete Attributes Expression</key>
</entry>
<entry>
<key>Store State</key>
<value>Do not store state</value>
</entry>
<entry>
<key>Stateful Variables Initial Value</key>
</entry>
<entry>
<key>target.node</key>
<value>${target.n:equals(0):ifElse('node1',
${target.n:equals(1):ifElse('node2',
${target.n:equals(2):ifElse('node3', 'n/a')}
)})}</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>Set target.node before send</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type>
</processors>
<processors>
<id>ec5cb9c9-8b28-3568-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<position>
<x>1554.0</x>
<y>-27.0</y>
</position>
<bundle>
<artifact>nifi-update-attribute-nar</artifact>
<group>org.apache.nifi</group>
<version>1.7.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Delete Attributes Expression</key>
<value>
<name>Delete Attributes Expression</name>
</value>
</entry>
<entry>
<key>Store State</key>
<value>
<name>Store State</name>
</value>
</entry>
<entry>
<key>Stateful Variables Initial Value</key>
<value>
<name>Stateful Variables Initial Value</name>
</value>
</entry>
<entry>
<key>target.n</key>
<value>
<name>target.n</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Delete Attributes Expression</key>
</entry>
<entry>
<key>Store State</key>
<value>Do not store state</value>
</entry>
<entry>
<key>Stateful Variables Initial Value</key>
</entry>
<entry>
<key>target.n</key>
<value>${random():mod(3)}</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>Set target.n before split</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type>
</processors>
<processors>
<id>edb5bf2c-1b6b-317b-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<position>
<x>2024.0</x>
<y>389.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.7.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>URL</key>
<value>
<name>URL</name>
</value>
</entry>
<entry>
<key>Max Batch Size</key>
<value>
<name>Max Batch Size</name>
</value>
</entry>
<entry>
<key>Max Data to Post per Second</key>
<value>
<name>Max Data to Post per Second</name>
</value>
</entry>
<entry>
<key>SSL Context Service</key>
<value>
<identifiesControllerService>org.apache.nifi.ssl.SSLContextService</identifiesControllerService>
<name>SSL Context Service</name>
</value>
</entry>
<entry>
<key>Username</key>
<value>
<name>Username</name>
</value>
</entry>
<entry>
<key>Password</key>
<value>
<name>Password</name>
</value>
</entry>
<entry>
<key>Send as FlowFile</key>
<value>
<name>Send as FlowFile</name>
</value>
</entry>
<entry>
<key>Use Chunked Encoding</key>
<value>
<name>Use Chunked Encoding</name>
</value>
</entry>
<entry>
<key>Compression Level</key>
<value>
<name>Compression Level</name>
</value>
</entry>
<entry>
<key>Connection Timeout</key>
<value>
<name>Connection Timeout</name>
</value>
</entry>
<entry>
<key>Data Timeout</key>
<value>
<name>Data Timeout</name>
</value>
</entry>
<entry>
<key>Attributes to Send as HTTP Headers (Regex)</key>
<value>
<name>Attributes to Send as HTTP Headers (Regex)</name>
</value>
</entry>
<entry>
<key>User Agent</key>
<value>
<name>User Agent</name>
</value>
</entry>
<entry>
<key>Proxy Host</key>
<value>
<name>Proxy Host</name>
</value>
</entry>
<entry>
<key>Proxy Port</key>
<value>
<name>Proxy Port</name>
</value>
</entry>
<entry>
<key>Content-Type</key>
<value>
<name>Content-Type</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>URL</key>
<value>${target.node}</value>
</entry>
<entry>
<key>Max Batch Size</key>
<value>100 MB</value>
</entry>
<entry>
<key>Max Data to Post per Second</key>
</entry>
<entry>
<key>SSL Context Service</key>
</entry>
<entry>
<key>Username</key>
</entry>
<entry>
<key>Password</key>
</entry>
<entry>
<key>Send as FlowFile</key>
<value>true</value>
</entry>
<entry>
<key>Use Chunked Encoding</key>
</entry>
<entry>
<key>Compression Level</key>
<value>0</value>
</entry>
<entry>
<key>Connection Timeout</key>
<value>30 sec</value>
</entry>
<entry>
<key>Data Timeout</key>
<value>30 sec</value>
</entry>
<entry>
<key>Attributes to Send as HTTP Headers (Regex)</key>
</entry>
<entry>
<key>User Agent</key>
<value>Apache-HttpClient/4.5.5 (Java/1.8.0_111)</value>
</entry>
<entry>
<key>Proxy Host</key>
</entry>
<entry>
<key>Proxy Port</key>
</entry>
<entry>
<key>Content-Type</key>
<value>${mime.type}</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>PostHTTP</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.PostHTTP</type>
</processors>
<processors>
<id>44ea26dc-eb7b-367b-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<position>
<x>2116.0</x>
<y>54.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.7.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Segment Size</key>
<value>
<name>Segment Size</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Segment Size</key>
<value>1b</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>SegmentContent</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>original</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>segments</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.standard.SegmentContent</type>
</processors>
<processors>
<id>47ee19c9-8807-3f55-0000-000000000000</id>
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId>
<position>
<x>1977.0</x>
<y>-227.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.7.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>File Size</key>
<value>
<name>File Size</name>
</value>
</entry>
<entry>
<key>Batch Size</key>
<value>
<name>Batch Size</name>
</value>
</entry>
<entry>
<key>Data Format</key>
<value>
<name>Data Format</name>
</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>
<name>Unique FlowFiles</name>
</value>
</entry>
<entry>
<key>generate-ff-custom-text</key>
<value>
<name>generate-ff-custom-text</name>
</value>
</entry>
<entry>
<key>character-set</key>
<value>
<name>character-set</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>File Size</key>
<value>0B</value>
</entry>
<entry>
<key>Batch Size</key>
<value>1</value>
</entry>
<entry>
<key>Data Format</key>
<value>Text</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>false</value>
</entry>
<entry>
<key>generate-ff-custom-text</key>
<value>123</value>
</entry>
<entry>
<key>character-set</key>
<value>UTF-8</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>1d</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>GenerateFlowFile</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.standard.GenerateFlowFile</type>
</processors>
</contents>
<name>Large-S2S-workaround</name>
<variables/>
</processGroups>
</snippet>
<timestamp>04/10/2018 11:59:57 JST</timestamp>
</template>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.