Last active
August 15, 2021 15:52
-
-
Save ijokarumawak/077d7fdca57b9c8ff386f28c5198efd1 to your computer and use it in GitHub Desktop.
NiFi, A workaround if sending large data via S2S does not work due to taking more than the default 'idle connection expiration' (30s)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?xml version="1.0" encoding="UTF-8" standalone="yes"?> | |
| <template encoding-version="1.2"> | |
| <description></description> | |
| <groupId>aa0b8252-0162-1000-87e0-8c2881d45fa1</groupId> | |
| <name>Large S2S Workaround</name> | |
| <snippet> | |
| <processGroups> | |
| <id>59d79054-2944-3b82-0000-000000000000</id> | |
| <parentGroupId>0329e4dc-ff21-384f-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>0.0</x> | |
| <y>0.0</y> | |
| </position> | |
| <comments></comments> | |
| <contents> | |
| <connections> | |
| <id>81d2128b-ae34-3105-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>8170b2d5-b073-361a-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>segments</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>abd350ed-4b4c-3aeb-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>b1980502-9d12-32fb-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>edb5bf2c-1b6b-317b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>8170b2d5-b073-361a-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>7f412446-dc22-313e-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59d79054-2944-3b82-0000-000000000000</groupId> | |
| <id>47ee19c9-8807-3f55-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <labels> | |
| <id>d7cf9dc5-f685-30e7-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1943.0</x> | |
| <y>251.0</y> | |
| </position> | |
| <height>53.0</height> | |
| <label>Segments created from the same FlowFile | |
| will have the same target.node attribute.</label> | |
| <style> | |
| <entry> | |
| <key>font-size</key> | |
| <value>12px</value> | |
| </entry> | |
| </style> | |
| <width>267.0</width> | |
| </labels> | |
| <processors> | |
| <id>8170b2d5-b073-361a-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1543.0</x> | |
| <y>204.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-update-attribute-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| <value> | |
| <name>Delete Attributes Expression</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value> | |
| <name>Store State</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| <value> | |
| <name>Stateful Variables Initial Value</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>target.node</key> | |
| <value> | |
| <name>target.node</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value>Do not store state</value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| </entry> | |
| <entry> | |
| <key>target.node</key> | |
| <value>${target.n:equals(0):ifElse('node1', | |
| ${target.n:equals(1):ifElse('node2', | |
| ${target.n:equals(2):ifElse('node3', 'n/a')} | |
| )})}</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>Set target.node before send</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
| </processors> | |
| <processors> | |
| <id>ec5cb9c9-8b28-3568-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1554.0</x> | |
| <y>-27.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-update-attribute-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| <value> | |
| <name>Delete Attributes Expression</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value> | |
| <name>Store State</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| <value> | |
| <name>Stateful Variables Initial Value</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>target.n</key> | |
| <value> | |
| <name>target.n</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value>Do not store state</value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| </entry> | |
| <entry> | |
| <key>target.n</key> | |
| <value>${random():mod(3)}</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>Set target.n before split</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
| </processors> | |
| <processors> | |
| <id>edb5bf2c-1b6b-317b-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>2024.0</x> | |
| <y>389.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>URL</key> | |
| <value> | |
| <name>URL</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Max Batch Size</key> | |
| <value> | |
| <name>Max Batch Size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Max Data to Post per Second</key> | |
| <value> | |
| <name>Max Data to Post per Second</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>SSL Context Service</key> | |
| <value> | |
| <identifiesControllerService>org.apache.nifi.ssl.SSLContextService</identifiesControllerService> | |
| <name>SSL Context Service</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Username</key> | |
| <value> | |
| <name>Username</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Password</key> | |
| <value> | |
| <name>Password</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Send as FlowFile</key> | |
| <value> | |
| <name>Send as FlowFile</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Use Chunked Encoding</key> | |
| <value> | |
| <name>Use Chunked Encoding</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Compression Level</key> | |
| <value> | |
| <name>Compression Level</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Connection Timeout</key> | |
| <value> | |
| <name>Connection Timeout</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Data Timeout</key> | |
| <value> | |
| <name>Data Timeout</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Attributes to Send as HTTP Headers (Regex)</key> | |
| <value> | |
| <name>Attributes to Send as HTTP Headers (Regex)</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>User Agent</key> | |
| <value> | |
| <name>User Agent</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Host</key> | |
| <value> | |
| <name>Proxy Host</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Port</key> | |
| <value> | |
| <name>Proxy Port</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Content-Type</key> | |
| <value> | |
| <name>Content-Type</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>URL</key> | |
| <value>${target.node}</value> | |
| </entry> | |
| <entry> | |
| <key>Max Batch Size</key> | |
| <value>100 MB</value> | |
| </entry> | |
| <entry> | |
| <key>Max Data to Post per Second</key> | |
| </entry> | |
| <entry> | |
| <key>SSL Context Service</key> | |
| </entry> | |
| <entry> | |
| <key>Username</key> | |
| </entry> | |
| <entry> | |
| <key>Password</key> | |
| </entry> | |
| <entry> | |
| <key>Send as FlowFile</key> | |
| <value>true</value> | |
| </entry> | |
| <entry> | |
| <key>Use Chunked Encoding</key> | |
| </entry> | |
| <entry> | |
| <key>Compression Level</key> | |
| <value>0</value> | |
| </entry> | |
| <entry> | |
| <key>Connection Timeout</key> | |
| <value>30 sec</value> | |
| </entry> | |
| <entry> | |
| <key>Data Timeout</key> | |
| <value>30 sec</value> | |
| </entry> | |
| <entry> | |
| <key>Attributes to Send as HTTP Headers (Regex)</key> | |
| </entry> | |
| <entry> | |
| <key>User Agent</key> | |
| <value>Apache-HttpClient/4.5.5 (Java/1.8.0_111)</value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Host</key> | |
| </entry> | |
| <entry> | |
| <key>Proxy Port</key> | |
| </entry> | |
| <entry> | |
| <key>Content-Type</key> | |
| <value>${mime.type}</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>PostHTTP</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>failure</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>STOPPED</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.PostHTTP</type> | |
| </processors> | |
| <processors> | |
| <id>44ea26dc-eb7b-367b-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>2116.0</x> | |
| <y>54.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Segment Size</key> | |
| <value> | |
| <name>Segment Size</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Segment Size</key> | |
| <value>1b</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>SegmentContent</name> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>original</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>segments</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.SegmentContent</type> | |
| </processors> | |
| <processors> | |
| <id>47ee19c9-8807-3f55-0000-000000000000</id> | |
| <parentGroupId>59d79054-2944-3b82-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>1977.0</x> | |
| <y>-227.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.7.0-SNAPSHOT</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>File Size</key> | |
| <value> | |
| <name>File Size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Batch Size</key> | |
| <value> | |
| <name>Batch Size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Data Format</key> | |
| <value> | |
| <name>Data Format</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Unique FlowFiles</key> | |
| <value> | |
| <name>Unique FlowFiles</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>generate-ff-custom-text</key> | |
| <value> | |
| <name>generate-ff-custom-text</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>character-set</key> | |
| <value> | |
| <name>character-set</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>File Size</key> | |
| <value>0B</value> | |
| </entry> | |
| <entry> | |
| <key>Batch Size</key> | |
| <value>1</value> | |
| </entry> | |
| <entry> | |
| <key>Data Format</key> | |
| <value>Text</value> | |
| </entry> | |
| <entry> | |
| <key>Unique FlowFiles</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>generate-ff-custom-text</key> | |
| <value>123</value> | |
| </entry> | |
| <entry> | |
| <key>character-set</key> | |
| <value>UTF-8</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>1d</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <name>GenerateFlowFile</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>RUNNING</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> | |
| </processors> | |
| </contents> | |
| <name>Large-S2S-workaround</name> | |
| <variables/> | |
| </processGroups> | |
| </snippet> | |
| <timestamp>04/10/2018 11:59:57 JST</timestamp> | |
| </template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment