Created
January 28, 2020 13:40
-
-
Save SergiyKolesnikov/61938ac351d46b984522e9346c6e79e8 to your computer and use it in GitHub Desktop.
A NiFi flow template that uses REST API as a data source and local filesystem as a data sink. The flow calls https://api.exchangeratesapi.io/latest every 10 minutes and saves the response in a file under /tmp/data-out.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?xml version="1.0" encoding="UTF-8" standalone="yes"?> | |
| <template encoding-version="1.3"> | |
| <description>ETL example using a REST API</description> | |
| <groupId>e69b77ca-016f-1000-02b1-0af72eced378</groupId> | |
| <name>nifi_rest_api_etl</name> | |
| <snippet> | |
| <connections> | |
| <id>829c718c-f0af-3504-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59eaeeb3-00b4-3342-0000-000000000000</groupId> | |
| <id>6f51eda0-9390-350c-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
| <loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
| <loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
| <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
| <name></name> | |
| <selectedRelationships>Response</selectedRelationships> | |
| <source> | |
| <groupId>59eaeeb3-00b4-3342-0000-000000000000</groupId> | |
| <id>bbc01c1b-f016-38d9-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <connections> | |
| <id>91377bd0-11f7-3dea-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
| <backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
| <destination> | |
| <groupId>59eaeeb3-00b4-3342-0000-000000000000</groupId> | |
| <id>39dc4831-7105-3c7b-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </destination> | |
| <flowFileExpiration>0 sec</flowFileExpiration> | |
| <labelIndex>1</labelIndex> | |
| <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
| <loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
| <loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
| <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
| <name></name> | |
| <selectedRelationships>success</selectedRelationships> | |
| <source> | |
| <groupId>59eaeeb3-00b4-3342-0000-000000000000</groupId> | |
| <id>6f51eda0-9390-350c-0000-000000000000</id> | |
| <type>PROCESSOR</type> | |
| </source> | |
| <zIndex>0</zIndex> | |
| </connections> | |
| <labels> | |
| <id>64512057-d56e-382c-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>0.0</x> | |
| <y>0.0</y> | |
| </position> | |
| <height>24.0</height> | |
| <label>1</label> | |
| <style> | |
| <entry> | |
| <key>font-size</key> | |
| <value>12px</value> | |
| </entry> | |
| </style> | |
| <width>64.0</width> | |
| </labels> | |
| <labels> | |
| <id>a413f150-f829-34e7-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>608.0</x> | |
| <y>200.0</y> | |
| </position> | |
| <height>24.0</height> | |
| <label>3</label> | |
| <style> | |
| <entry> | |
| <key>font-size</key> | |
| <value>12px</value> | |
| </entry> | |
| </style> | |
| <width>64.0</width> | |
| </labels> | |
| <labels> | |
| <id>b41e15c8-59bf-3dea-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>0.0</x> | |
| <y>200.0</y> | |
| </position> | |
| <height>24.0</height> | |
| <label>2</label> | |
| <style> | |
| <entry> | |
| <key>font-size</key> | |
| <value>12px</value> | |
| </entry> | |
| </style> | |
| <width>64.0</width> | |
| </labels> | |
| <processors> | |
| <id>39dc4831-7105-3c7b-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>608.0</x> | |
| <y>232.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.11.0</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Directory</key> | |
| <value> | |
| <name>Directory</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Conflict Resolution Strategy</key> | |
| <value> | |
| <name>Conflict Resolution Strategy</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Create Missing Directories</key> | |
| <value> | |
| <name>Create Missing Directories</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Maximum File Count</key> | |
| <value> | |
| <name>Maximum File Count</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Last Modified Time</key> | |
| <value> | |
| <name>Last Modified Time</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Permissions</key> | |
| <value> | |
| <name>Permissions</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Owner</key> | |
| <value> | |
| <name>Owner</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Group</key> | |
| <value> | |
| <name>Group</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Directory</key> | |
| <value>/tmp/data-out/</value> | |
| </entry> | |
| <entry> | |
| <key>Conflict Resolution Strategy</key> | |
| <value>fail</value> | |
| </entry> | |
| <entry> | |
| <key>Create Missing Directories</key> | |
| <value>true</value> | |
| </entry> | |
| <entry> | |
| <key>Maximum File Count</key> | |
| </entry> | |
| <entry> | |
| <key>Last Modified Time</key> | |
| </entry> | |
| <entry> | |
| <key>Permissions</key> | |
| </entry> | |
| <entry> | |
| <key>Owner</key> | |
| </entry> | |
| <entry> | |
| <key>Group</key> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <executionNodeRestricted>false</executionNodeRestricted> | |
| <name>Save the output file</name> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>failure</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>STOPPED</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.PutFile</type> | |
| </processors> | |
| <processors> | |
| <id>6f51eda0-9390-350c-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>0.0</x> | |
| <y>232.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-update-attribute-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.11.0</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| <value> | |
| <name>Delete Attributes Expression</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value> | |
| <name>Store State</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| <value> | |
| <name>Stateful Variables Initial Value</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>canonical-value-lookup-cache-size</key> | |
| <value> | |
| <name>canonical-value-lookup-cache-size</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>filename</key> | |
| <value> | |
| <name>filename</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>Delete Attributes Expression</key> | |
| </entry> | |
| <entry> | |
| <key>Store State</key> | |
| <value>Do not store state</value> | |
| </entry> | |
| <entry> | |
| <key>Stateful Variables Initial Value</key> | |
| </entry> | |
| <entry> | |
| <key>canonical-value-lookup-cache-size</key> | |
| <value>100</value> | |
| </entry> | |
| <entry> | |
| <key>filename</key> | |
| <value>exchangerates_${filename}.json</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>0 sec</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <executionNodeRestricted>false</executionNodeRestricted> | |
| <name>Set the output-file name</name> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>success</name> | |
| </relationships> | |
| <state>STOPPED</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
| </processors> | |
| <processors> | |
| <id>bbc01c1b-f016-38d9-0000-000000000000</id> | |
| <parentGroupId>59eaeeb3-00b4-3342-0000-000000000000</parentGroupId> | |
| <position> | |
| <x>0.0</x> | |
| <y>32.0</y> | |
| </position> | |
| <bundle> | |
| <artifact>nifi-standard-nar</artifact> | |
| <group>org.apache.nifi</group> | |
| <version>1.11.0</version> | |
| </bundle> | |
| <config> | |
| <bulletinLevel>WARN</bulletinLevel> | |
| <comments></comments> | |
| <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
| <descriptors> | |
| <entry> | |
| <key>HTTP Method</key> | |
| <value> | |
| <name>HTTP Method</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Remote URL</key> | |
| <value> | |
| <name>Remote URL</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>SSL Context Service</key> | |
| <value> | |
| <identifiesControllerService>org.apache.nifi.ssl.SSLContextService</identifiesControllerService> | |
| <name>SSL Context Service</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Connection Timeout</key> | |
| <value> | |
| <name>Connection Timeout</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Read Timeout</key> | |
| <value> | |
| <name>Read Timeout</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Include Date Header</key> | |
| <value> | |
| <name>Include Date Header</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Follow Redirects</key> | |
| <value> | |
| <name>Follow Redirects</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Attributes to Send</key> | |
| <value> | |
| <name>Attributes to Send</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Basic Authentication Username</key> | |
| <value> | |
| <name>Basic Authentication Username</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Basic Authentication Password</key> | |
| <value> | |
| <name>Basic Authentication Password</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>proxy-configuration-service</key> | |
| <value> | |
| <identifiesControllerService>org.apache.nifi.proxy.ProxyConfigurationService</identifiesControllerService> | |
| <name>proxy-configuration-service</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Host</key> | |
| <value> | |
| <name>Proxy Host</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Port</key> | |
| <value> | |
| <name>Proxy Port</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Proxy Type</key> | |
| <value> | |
| <name>Proxy Type</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>invokehttp-proxy-user</key> | |
| <value> | |
| <name>invokehttp-proxy-user</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>invokehttp-proxy-password</key> | |
| <value> | |
| <name>invokehttp-proxy-password</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Put Response Body In Attribute</key> | |
| <value> | |
| <name>Put Response Body In Attribute</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Max Length To Put In Attribute</key> | |
| <value> | |
| <name>Max Length To Put In Attribute</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Digest Authentication</key> | |
| <value> | |
| <name>Digest Authentication</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Always Output Response</key> | |
| <value> | |
| <name>Always Output Response</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Add Response Headers to Request</key> | |
| <value> | |
| <name>Add Response Headers to Request</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Content-Type</key> | |
| <value> | |
| <name>Content-Type</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>send-message-body</key> | |
| <value> | |
| <name>send-message-body</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Use Chunked Encoding</key> | |
| <value> | |
| <name>Use Chunked Encoding</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>Penalize on "No Retry"</key> | |
| <value> | |
| <name>Penalize on "No Retry"</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>use-etag</key> | |
| <value> | |
| <name>use-etag</name> | |
| </value> | |
| </entry> | |
| <entry> | |
| <key>etag-max-cache-size</key> | |
| <value> | |
| <name>etag-max-cache-size</name> | |
| </value> | |
| </entry> | |
| </descriptors> | |
| <executionNode>ALL</executionNode> | |
| <lossTolerant>false</lossTolerant> | |
| <penaltyDuration>30 sec</penaltyDuration> | |
| <properties> | |
| <entry> | |
| <key>HTTP Method</key> | |
| <value>GET</value> | |
| </entry> | |
| <entry> | |
| <key>Remote URL</key> | |
| <value>https://api.exchangeratesapi.io/latest</value> | |
| </entry> | |
| <entry> | |
| <key>SSL Context Service</key> | |
| </entry> | |
| <entry> | |
| <key>Connection Timeout</key> | |
| <value>5 secs</value> | |
| </entry> | |
| <entry> | |
| <key>Read Timeout</key> | |
| <value>15 secs</value> | |
| </entry> | |
| <entry> | |
| <key>Include Date Header</key> | |
| <value>True</value> | |
| </entry> | |
| <entry> | |
| <key>Follow Redirects</key> | |
| <value>True</value> | |
| </entry> | |
| <entry> | |
| <key>Attributes to Send</key> | |
| </entry> | |
| <entry> | |
| <key>Basic Authentication Username</key> | |
| </entry> | |
| <entry> | |
| <key>Basic Authentication Password</key> | |
| </entry> | |
| <entry> | |
| <key>proxy-configuration-service</key> | |
| </entry> | |
| <entry> | |
| <key>Proxy Host</key> | |
| </entry> | |
| <entry> | |
| <key>Proxy Port</key> | |
| </entry> | |
| <entry> | |
| <key>Proxy Type</key> | |
| <value>http</value> | |
| </entry> | |
| <entry> | |
| <key>invokehttp-proxy-user</key> | |
| </entry> | |
| <entry> | |
| <key>invokehttp-proxy-password</key> | |
| </entry> | |
| <entry> | |
| <key>Put Response Body In Attribute</key> | |
| </entry> | |
| <entry> | |
| <key>Max Length To Put In Attribute</key> | |
| <value>256</value> | |
| </entry> | |
| <entry> | |
| <key>Digest Authentication</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>Always Output Response</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>Add Response Headers to Request</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>Content-Type</key> | |
| <value>${mime.type}</value> | |
| </entry> | |
| <entry> | |
| <key>send-message-body</key> | |
| <value>true</value> | |
| </entry> | |
| <entry> | |
| <key>Use Chunked Encoding</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>Penalize on "No Retry"</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>use-etag</key> | |
| <value>false</value> | |
| </entry> | |
| <entry> | |
| <key>etag-max-cache-size</key> | |
| <value>10MB</value> | |
| </entry> | |
| </properties> | |
| <runDurationMillis>0</runDurationMillis> | |
| <schedulingPeriod>10 min</schedulingPeriod> | |
| <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
| <yieldDuration>1 sec</yieldDuration> | |
| </config> | |
| <executionNodeRestricted>false</executionNodeRestricted> | |
| <name>Query the REST API</name> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>Failure</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>No Retry</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>Original</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>false</autoTerminate> | |
| <name>Response</name> | |
| </relationships> | |
| <relationships> | |
| <autoTerminate>true</autoTerminate> | |
| <name>Retry</name> | |
| </relationships> | |
| <state>STOPPED</state> | |
| <style/> | |
| <type>org.apache.nifi.processors.standard.InvokeHTTP</type> | |
| </processors> | |
| </snippet> | |
| <timestamp>01/28/2020 14:37:14 CET</timestamp> | |
| </template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment