Instantly share code, notes, and snippets.

Embed
What would you like to do?
<?xml version="1.0" ?>
<template encoding-version="1.0">
<description></description>
<groupId>c3137542-0157-1000-a51c-b70d9a98dafa</groupId>
<name>UsingApacheNiFiToIngestSNMPTablesIntoAvro</name>
<snippet>
<connections>
<id>d284dac3-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d284bfb6-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2797d9f-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>d2a5389b-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2a515ce-0157-1000-0000-000000000000</id>
<type>FUNNEL</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>output stream</selectedRelationships>
<source>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d284bfb6-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>d2bf2387-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2be5192-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<source>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2a515ce-0157-1000-0000-000000000000</id>
<type>FUNNEL</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>d3b95af6-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2bf4c2b-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2be5192-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>d3be9cda-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d3be81ac-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d2bf4c2b-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>d3ce5e86-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d3ce2802-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>c3137542-0157-1000-0000-000000000000</groupId>
<id>d3be81ac-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<funnels>
<id>d2a515ce-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>146.00003051757812</x>
<y>433.0</y>
</position>
</funnels>
<processors>
<id>d2797d9f-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>4.000030517578125</x>
<y>0.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>File Size</key>
<value>
<name>File Size</name>
</value>
</entry>
<entry>
<key>Batch Size</key>
<value>
<name>Batch Size</name>
</value>
</entry>
<entry>
<key>Data Format</key>
<value>
<name>Data Format</name>
</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>
<name>Unique FlowFiles</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>File Size</key>
<value>1b</value>
</entry>
<entry>
<key>Batch Size</key>
<value>1</value>
</entry>
<entry>
<key>Data Format</key>
<value>Binary</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>false</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>1 min</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>GenerateFlowFile</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.GenerateFlowFile</type>
</processors>
<processors>
<id>d284bfb6-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>221.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Command Arguments</key>
<value>
<name>Command Arguments</name>
</value>
</entry>
<entry>
<key>Command Path</key>
<value>
<name>Command Path</name>
</value>
</entry>
<entry>
<key>Ignore STDIN</key>
<value>
<name>Ignore STDIN</name>
</value>
</entry>
<entry>
<key>Working Directory</key>
<value>
<name>Working Directory</name>
</value>
</entry>
<entry>
<key>Argument Delimiter</key>
<value>
<name>Argument Delimiter</name>
</value>
</entry>
<entry>
<key>Output Destination Attribute</key>
<value>
<name>Output Destination Attribute</name>
</value>
</entry>
<entry>
<key>Max Attribute Length</key>
<value>
<name>Max Attribute Length</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Command Arguments</key>
<value>-Cf ; -v 2c -c 127.0.0.1 127.0.0.1:1610 ifTable</value>
</entry>
<entry>
<key>Command Path</key>
<value>/usr/bin/snmptable</value>
</entry>
<entry>
<key>Ignore STDIN</key>
<value>false</value>
</entry>
<entry>
<key>Working Directory</key>
</entry>
<entry>
<key>Argument Delimiter</key>
<value> </value>
</entry>
<entry>
<key>Output Destination Attribute</key>
</entry>
<entry>
<key>Max Attribute Length</key>
<value>256</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ExecuteStreamCommand</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>original</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>output stream</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type>
</processors>
<processors>
<id>d2be5192-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>11.0</x>
<y>538.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Regular Expression</key>
<value>
<name>Regular Expression</name>
</value>
</entry>
<entry>
<key>Replacement Value</key>
<value>
<name>Replacement Value</name>
</value>
</entry>
<entry>
<key>Character Set</key>
<value>
<name>Character Set</name>
</value>
</entry>
<entry>
<key>Maximum Buffer Size</key>
<value>
<name>Maximum Buffer Size</name>
</value>
</entry>
<entry>
<key>Replacement Strategy</key>
<value>
<name>Replacement Strategy</name>
</value>
</entry>
<entry>
<key>Evaluation Mode</key>
<value>
<name>Evaluation Mode</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Regular Expression</key>
<value>^SNMP table: (.+)\n\n</value>
</entry>
<entry>
<key>Replacement Value</key>
<value></value>
</entry>
<entry>
<key>Character Set</key>
<value>UTF-8</value>
</entry>
<entry>
<key>Maximum Buffer Size</key>
<value>1 MB</value>
</entry>
<entry>
<key>Replacement Strategy</key>
<value>Regex Replace</value>
</entry>
<entry>
<key>Evaluation Mode</key>
<value>Entire text</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ReplaceText</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.ReplaceText</type>
</processors>
<processors>
<id>d2bf4c2b-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>15.0</x>
<y>768.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Schema Output Destination</key>
<value>
<name>Schema Output Destination</name>
</value>
</entry>
<entry>
<key>Input Content Type</key>
<value>
<name>Input Content Type</name>
</value>
</entry>
<entry>
<key>CSV Header Definition</key>
<value>
<name>CSV Header Definition</name>
</value>
</entry>
<entry>
<key>Get CSV Header Definition From Data</key>
<value>
<name>Get CSV Header Definition From Data</name>
</value>
</entry>
<entry>
<key>CSV Header Line Skip Count</key>
<value>
<name>CSV Header Line Skip Count</name>
</value>
</entry>
<entry>
<key>CSV delimiter</key>
<value>
<name>CSV delimiter</name>
</value>
</entry>
<entry>
<key>CSV Escape String</key>
<value>
<name>CSV Escape String</name>
</value>
</entry>
<entry>
<key>CSV Quote String</key>
<value>
<name>CSV Quote String</name>
</value>
</entry>
<entry>
<key>Pretty Avro Output</key>
<value>
<name>Pretty Avro Output</name>
</value>
</entry>
<entry>
<key>Avro Record Name</key>
<value>
<name>Avro Record Name</name>
</value>
</entry>
<entry>
<key>Number Of Records To Analyze</key>
<value>
<name>Number Of Records To Analyze</name>
</value>
</entry>
<entry>
<key>Charset</key>
<value>
<name>Charset</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Schema Output Destination</key>
<value>flowfile-attribute</value>
</entry>
<entry>
<key>Input Content Type</key>
<value>csv</value>
</entry>
<entry>
<key>CSV Header Definition</key>
</entry>
<entry>
<key>Get CSV Header Definition From Data</key>
<value>true</value>
</entry>
<entry>
<key>CSV Header Line Skip Count</key>
<value>0</value>
</entry>
<entry>
<key>CSV delimiter</key>
<value>;</value>
</entry>
<entry>
<key>CSV Escape String</key>
<value>\</value>
</entry>
<entry>
<key>CSV Quote String</key>
<value>'</value>
</entry>
<entry>
<key>Pretty Avro Output</key>
<value>true</value>
</entry>
<entry>
<key>Avro Record Name</key>
<value>iftablerow</value>
</entry>
<entry>
<key>Number Of Records To Analyze</key>
<value>10</value>
</entry>
<entry>
<key>Charset</key>
<value>UTF-8</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>InferAvroSchema</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>original</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>unsupported content</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.kite.InferAvroSchema</type>
</processors>
<processors>
<id>d3be81ac-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>550.3464555193742</x>
<y>773.2506118701741</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Hadoop configuration files</key>
<value>
<name>Hadoop configuration files</name>
</value>
</entry>
<entry>
<key>Record schema</key>
<value>
<name>Record schema</name>
</value>
</entry>
<entry>
<key>CSV charset</key>
<value>
<name>CSV charset</name>
</value>
</entry>
<entry>
<key>CSV delimiter</key>
<value>
<name>CSV delimiter</name>
</value>
</entry>
<entry>
<key>CSV quote character</key>
<value>
<name>CSV quote character</name>
</value>
</entry>
<entry>
<key>CSV escape character</key>
<value>
<name>CSV escape character</name>
</value>
</entry>
<entry>
<key>Use CSV header line</key>
<value>
<name>Use CSV header line</name>
</value>
</entry>
<entry>
<key>Lines to skip</key>
<value>
<name>Lines to skip</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Hadoop configuration files</key>
</entry>
<entry>
<key>Record schema</key>
<value>${inferred.avro.schema}</value>
</entry>
<entry>
<key>CSV charset</key>
<value>utf8</value>
</entry>
<entry>
<key>CSV delimiter</key>
<value>;</value>
</entry>
<entry>
<key>CSV quote character</key>
<value>"</value>
</entry>
<entry>
<key>CSV escape character</key>
<value>\</value>
</entry>
<entry>
<key>Use CSV header line</key>
<value>false</value>
</entry>
<entry>
<key>Lines to skip</key>
<value>0</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ConvertCSVToAvro</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>incompatible</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.kite.ConvertCSVToAvro</type>
</processors>
<processors>
<id>d3ce2802-0157-1000-0000-000000000000</id>
<parentGroupId>c3137542-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>548.6993907296755</x>
<y>555.8225187022117</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Directory</key>
<value>
<name>Directory</name>
</value>
</entry>
<entry>
<key>Conflict Resolution Strategy</key>
<value>
<name>Conflict Resolution Strategy</name>
</value>
</entry>
<entry>
<key>Create Missing Directories</key>
<value>
<name>Create Missing Directories</name>
</value>
</entry>
<entry>
<key>Maximum File Count</key>
<value>
<name>Maximum File Count</name>
</value>
</entry>
<entry>
<key>Last Modified Time</key>
<value>
<name>Last Modified Time</name>
</value>
</entry>
<entry>
<key>Permissions</key>
<value>
<name>Permissions</name>
</value>
</entry>
<entry>
<key>Owner</key>
<value>
<name>Owner</name>
</value>
</entry>
<entry>
<key>Group</key>
<value>
<name>Group</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Directory</key>
<value>/home/ubuntu/nifi/data</value>
</entry>
<entry>
<key>Conflict Resolution Strategy</key>
<value>fail</value>
</entry>
<entry>
<key>Create Missing Directories</key>
<value>true</value>
</entry>
<entry>
<key>Maximum File Count</key>
</entry>
<entry>
<key>Last Modified Time</key>
</entry>
<entry>
<key>Permissions</key>
</entry>
<entry>
<key>Owner</key>
</entry>
<entry>
<key>Group</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>PutFile</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.PutFile</type>
</processors>
</snippet>
<timestamp>10/17/2016 21:04:16 UTC</timestamp>
</template>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment