NiFi, A workaround if sending large data via S2S does not work due to taking more than the default 'idle connection expiration' (30s)
| <?xml version="1.0" encoding="UTF-8" standalone="yes"?> | |
| <template encoding-version="1.2"> | |
| <description></description> | |
| <groupId>aa0b8252-0162-1000-87e0-8c2881d45fa1</groupId> | |
| <name>Large S2S Workaround</name> | |
| <snippet> | |
| <processGroups> | |
| <id>59d79054-2944-3b82-0000-000000000000</id> | |
| <parentGroupId>0329e4dc-ff21-384f-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>0.0</x> | |
| <y>0.0</y> | |
| </position> | |
| <comments></comments> | |
| <contents> | |
| <connections> | |
| <id>81d2128b-ae34-3105-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>8170b2d5-b073-361a-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>segments</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>abd350ed-4b4c-3aeb-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>b1980502-9d12-32fb-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>edb5bf2c-1b6b-317b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>8170b2d5-b073-361a-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>7f412446-dc22-313e-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>47ee19c9-8807-3f55-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <labels> | |
| <id>d7cf9dc5-f685-30e7-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1943.0</x> | |
| <y>251.0</y> | |
| </position> | |
| <height>53.0</height> | |
| <label>Segments created from the same FlowFile | |
| will have the same target.node attribute.</label> | |
| <style> | |
| <entry> | |
| <key>font-size</key> | |
| <value>12px</value> | |
| </entry> | |
| </style> | |
| <width>267.0</width> | |
| </labels> | |
| <processors> | |
| <id>8170b2d5-b073-361a-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1543.0</x> | |
| <y>204.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-update-attribute-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| <value> | |
| <name>Delete Attributes Expression</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value> | |
| <name>Store State</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| <value> | |
| <name>Stateful Variables Initial Value</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>target.node</key> | |
| <value> | |
| <name>target.node</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value>Do not store state</value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| </entry> | |
| <entry> | |
| <key>target.node</key> | |
| <value>${target.n:equals(0):ifElse('node1', | |
| ${target.n:equals(1):ifElse('node2', | |
| ${target.n:equals(2):ifElse('node3', 'n/a')} | |
| )})}</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>Set target.node before send</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
| </processors> | |
| <processors> | |
| <id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1554.0</x> | |
| <y>-27.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-update-attribute-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| <value> | |
| <name>Delete Attributes Expression</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value> | |
| <name>Store State</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| <value> | |
| <name>Stateful Variables Initial Value</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>target.n</key> | |
| <value> | |
| <name>target.n</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value>Do not store state</value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| </entry> | |
| <entry> | |
| <key>target.n</key> | |
| <value>${random():mod(3)}</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>Set target.n before split</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
| </processors> | |
| <processors> | |
| <id>edb5bf2c-1b6b-317b-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>2024.0</x> | |
| <y>389.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>URL</key> | |
| <value> | |
| <name>URL</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Max Batch Size</key> | |
| <value> | |
| <name>Max Batch Size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Max Data to Post per Second</key> | |
| <value> | |
| <name>Max Data to Post per Second</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>SSL Context Service</key> | |
| <value> | |
| <identifiesControllerService>org.apache.nifi.ssl.SSLContextService</identifiesControllerService> | |
| <name>SSL Context Service</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Username</key> | |
| <value> | |
| <name>Username</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Password</key> | |
| <value> | |
| <name>Password</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Send as FlowFile</key> | |
| <value> | |
| <name>Send as FlowFile</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Use Chunked Encoding</key> | |
| <value> | |
| <name>Use Chunked Encoding</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Compression Level</key> | |
| <value> | |
| <name>Compression Level</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Connection Timeout</key> | |
| <value> | |
| <name>Connection Timeout</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Data Timeout</key> | |
| <value> | |
| <name>Data Timeout</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Attributes to Send as HTTP Headers (Regex)</key> | |
| <value> | |
| <name>Attributes to Send as HTTP Headers (Regex)</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>User Agent</key> | |
| <value> | |
| <name>User Agent</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Host</key> | |
| <value> | |
| <name>Proxy Host</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Port</key> | |
| <value> | |
| <name>Proxy Port</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Content-Type</key> | |
| <value> | |
| <name>Content-Type</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>URL</key> | |
| <value>${target.node}</value> | |
| </entry> | |
| <entry> | |
| <key>Max Batch Size</key> | |
| <value>100 MB</value> | |
| </entry> | |
| <entry> | |
| <key>Max Data to Post per Second</key> | |
| </entry> | |
| <entry> | |
| <key>SSL Context Service</key> | |
| </entry> | |
| <entry> | |
| <key>Username</key> | |
| </entry> | |
| <entry> | |
| <key>Password</key> | |
| </entry> | |
| <entry> | |
| <key>Send as FlowFile</key> | |
| <value>true</value> | |
| </entry> | |
| <entry> | |
| <key>Use Chunked Encoding</key> | |
| </entry> | |
| <entry> | |
| <key>Compression Level</key> | |
| <value>0</value> | |
| </entry> | |
| <entry> | |
| <key>Connection Timeout</key> | |
| <value>30 sec</value> | |
| </entry> | |
| <entry> | |
| <key>Data Timeout</key> | |
| <value>30 sec</value> | |
| </entry> | |
| <entry> | |
| <key>Attributes to Send as HTTP Headers (Regex)</key> | |
| </entry> | |
| <entry> | |
| <key>User Agent</key> | |
| <value>Apache-HttpClient/4.5.5 (Java/1.8.0_111)</value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Host</key> | |
| </entry> | |
| <entry> | |
| <key>Proxy Port</key> | |
| </entry> | |
| <entry> | |
| <key>Content-Type</key> | |
| <value>${mime.type}</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>PostHTTP</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>failure</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>STOPPED</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.PostHTTP</type> | |
| </processors> | |
| <processors> | |
| <id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>2116.0</x> | |
| <y>54.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Segment Size</key> | |
| <value> | |
| <name>Segment Size</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Segment Size</key> | |
| <value>1b</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>SegmentContent</name> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>original</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>segments</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.SegmentContent</type> | |
| </processors> | |
| <processors> | |
| <id>47ee19c9-8807-3f55-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1977.0</x> | |
| <y>-227.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>File Size</key> | |
| <value> | |
| <name>File Size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Batch Size</key> | |
| <value> | |
| <name>Batch Size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Data Format</key> | |
| <value> | |
| <name>Data Format</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Unique FlowFiles</key> | |
| <value> | |
| <name>Unique FlowFiles</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>generate-ff-custom-text</key> | |
| <value> | |
| <name>generate-ff-custom-text</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>character-set</key> | |
| <value> | |
| <name>character-set</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>File Size</key> | |
| <value>0B</value> | |
| </entry> | |
| <entry> | |
| <key>Batch Size</key> | |
| <value>1</value> | |
| </entry> | |
| <entry> | |
| <key>Data Format</key> | |
| <value>Text</value> | |
| </entry> | |
| <entry> | |
| <key>Unique FlowFiles</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>generate-ff-custom-text</key> | |
| <value>123</value> | |
| </entry> | |
| <entry> | |
| <key>character-set</key> | |
| <value>UTF-8</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>1d</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>GenerateFlowFile</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> | |
| </processors> | |
| </contents> | |
| <name>Large-S2S-workaround</name> | |
| <variables/> | |
| </processGroups> | |
| </snippet> | |
| <timestamp>04/10/2018 11:59:57 JST</timestamp> | |
| </template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment