NiFi, A workaround if sending large data via S2S does not work due to taking more than the default 'idle connection expiration' (30s)
<?xml version="1.0" encoding="UTF-8" standalone="yes"?> | |
<template encoding-version="1.2"> | |
<description></description> | |
<groupId>aa0b8252-0162-1000-87e0-8c2881d45fa1</groupId> | |
<name>Large S2S Workaround</name> | |
<snippet> | |
<processGroups> | |
<id>59d79054-2944-3b82-0000-000000000000</id> | |
<parentGroupId>0329e4dc-ff21-384f-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.0</x> | |
<y>0.0</y> | |
</position> | |
<comments></comments> | |
<contents> | |
<connections> | |
<id>81d2128b-ae34-3105-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>8170b2d5-b073-361a-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>segments</selectedRelationships> | |
<source> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>abd350ed-4b4c-3aeb-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>b1980502-9d12-32fb-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>edb5bf2c-1b6b-317b-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>8170b2d5-b073-361a-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>7f412446-dc22-313e-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
<id>47ee19c9-8807-3f55-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<labels> | |
<id>d7cf9dc5-f685-30e7-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<position> | |
<x>1943.0</x> | |
<y>251.0</y> | |
</position> | |
<height>53.0</height> | |
<label>Segments created from the same FlowFile | |
will have the same target.node attribute.</label> | |
<style> | |
<entry> | |
<key>font-size</key> | |
<value>12px</value> | |
</entry> | |
</style> | |
<width>267.0</width> | |
</labels> | |
<processors> | |
<id>8170b2d5-b073-361a-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<position> | |
<x>1543.0</x> | |
<y>204.0</y> | |
</position> | |
<bundle> | |
<artifact>nifi-update-attribute-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.7.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
<value> | |
<name>Delete Attributes Expression</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value> | |
<name>Store State</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
<value> | |
<name>Stateful Variables Initial Value</name> | |
</value> | |
</entry> | |
<entry> | |
<key>target.node</key> | |
<value> | |
<name>target.node</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value>Do not store state</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
</entry> | |
<entry> | |
<key>target.node</key> | |
<value>${target.n:equals(0):ifElse('node1', | |
${target.n:equals(1):ifElse('node2', | |
${target.n:equals(2):ifElse('node3', 'n/a')} | |
)})}</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>Set target.node before send</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>RUNNING</state> | |
<style/> | |
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
</processors> | |
<processors> | |
<id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<position> | |
<x>1554.0</x> | |
<y>-27.0</y> | |
</position> | |
<bundle> | |
<artifact>nifi-update-attribute-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.7.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
<value> | |
<name>Delete Attributes Expression</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value> | |
<name>Store State</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
<value> | |
<name>Stateful Variables Initial Value</name> | |
</value> | |
</entry> | |
<entry> | |
<key>target.n</key> | |
<value> | |
<name>target.n</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value>Do not store state</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
</entry> | |
<entry> | |
<key>target.n</key> | |
<value>${random():mod(3)}</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>Set target.n before split</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>RUNNING</state> | |
<style/> | |
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
</processors> | |
<processors> | |
<id>edb5bf2c-1b6b-317b-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<position> | |
<x>2024.0</x> | |
<y>389.0</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.7.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>URL</key> | |
<value> | |
<name>URL</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Batch Size</key> | |
<value> | |
<name>Max Batch Size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Data to Post per Second</key> | |
<value> | |
<name>Max Data to Post per Second</name> | |
</value> | |
</entry> | |
<entry> | |
<key>SSL Context Service</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.ssl.SSLContextService</identifiesControllerService> | |
<name>SSL Context Service</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Username</key> | |
<value> | |
<name>Username</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Password</key> | |
<value> | |
<name>Password</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Send as FlowFile</key> | |
<value> | |
<name>Send as FlowFile</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Use Chunked Encoding</key> | |
<value> | |
<name>Use Chunked Encoding</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Compression Level</key> | |
<value> | |
<name>Compression Level</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Connection Timeout</key> | |
<value> | |
<name>Connection Timeout</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Data Timeout</key> | |
<value> | |
<name>Data Timeout</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Attributes to Send as HTTP Headers (Regex)</key> | |
<value> | |
<name>Attributes to Send as HTTP Headers (Regex)</name> | |
</value> | |
</entry> | |
<entry> | |
<key>User Agent</key> | |
<value> | |
<name>User Agent</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Proxy Host</key> | |
<value> | |
<name>Proxy Host</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Proxy Port</key> | |
<value> | |
<name>Proxy Port</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Content-Type</key> | |
<value> | |
<name>Content-Type</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>URL</key> | |
<value>${target.node}</value> | |
</entry> | |
<entry> | |
<key>Max Batch Size</key> | |
<value>100 MB</value> | |
</entry> | |
<entry> | |
<key>Max Data to Post per Second</key> | |
</entry> | |
<entry> | |
<key>SSL Context Service</key> | |
</entry> | |
<entry> | |
<key>Username</key> | |
</entry> | |
<entry> | |
<key>Password</key> | |
</entry> | |
<entry> | |
<key>Send as FlowFile</key> | |
<value>true</value> | |
</entry> | |
<entry> | |
<key>Use Chunked Encoding</key> | |
</entry> | |
<entry> | |
<key>Compression Level</key> | |
<value>0</value> | |
</entry> | |
<entry> | |
<key>Connection Timeout</key> | |
<value>30 sec</value> | |
</entry> | |
<entry> | |
<key>Data Timeout</key> | |
<value>30 sec</value> | |
</entry> | |
<entry> | |
<key>Attributes to Send as HTTP Headers (Regex)</key> | |
</entry> | |
<entry> | |
<key>User Agent</key> | |
<value>Apache-HttpClient/4.5.5 (Java/1.8.0_111)</value> | |
</entry> | |
<entry> | |
<key>Proxy Host</key> | |
</entry> | |
<entry> | |
<key>Proxy Port</key> | |
</entry> | |
<entry> | |
<key>Content-Type</key> | |
<value>${mime.type}</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>PostHTTP</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.PostHTTP</type> | |
</processors> | |
<processors> | |
<id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<position> | |
<x>2116.0</x> | |
<y>54.0</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.7.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Segment Size</key> | |
<value> | |
<name>Segment Size</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Segment Size</key> | |
<value>1b</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>SegmentContent</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>original</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>segments</name> | |
</relationships> | |
<state>RUNNING</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.SegmentContent</type> | |
</processors> | |
<processors> | |
<id>47ee19c9-8807-3f55-0000-000000000000</id> | |
<parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
<position> | |
<x>1977.0</x> | |
<y>-227.0</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.7.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>File Size</key> | |
<value> | |
<name>File Size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Batch Size</key> | |
<value> | |
<name>Batch Size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Data Format</key> | |
<value> | |
<name>Data Format</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Unique FlowFiles</key> | |
<value> | |
<name>Unique FlowFiles</name> | |
</value> | |
</entry> | |
<entry> | |
<key>generate-ff-custom-text</key> | |
<value> | |
<name>generate-ff-custom-text</name> | |
</value> | |
</entry> | |
<entry> | |
<key>character-set</key> | |
<value> | |
<name>character-set</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>File Size</key> | |
<value>0B</value> | |
</entry> | |
<entry> | |
<key>Batch Size</key> | |
<value>1</value> | |
</entry> | |
<entry> | |
<key>Data Format</key> | |
<value>Text</value> | |
</entry> | |
<entry> | |
<key>Unique FlowFiles</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>generate-ff-custom-text</key> | |
<value>123</value> | |
</entry> | |
<entry> | |
<key>character-set</key> | |
<value>UTF-8</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>1d</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>GenerateFlowFile</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>RUNNING</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.GenerateFlowFile</type> | |
</processors> | |
</contents> | |
<name>Large-S2S-workaround</name> | |
<variables/> | |
</processGroups> | |
</snippet> | |
<timestamp>04/10/2018 11:59:57 JST</timestamp> | |
</template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment