Skip to content

Instantly share code, notes, and snippets.

@alopresto

alopresto/extract_text_regex.xml

Created October 10, 2016 19:21
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save alopresto/f79604a8f0e803defac54b048d7a6b4d to your computer and use it in GitHub Desktop.
Save alopresto/f79604a8f0e803defac54b048d7a6b4d to your computer and use it in GitHub Desktop.
Download ZIP
Apache NiFi template which generates an empty flowfile, populates the contents with an example log HTTP request line, and extracts the HTTP response code.
Raw
extract_text_regex.xml
<?xml version="1.0" ?>
<template encoding-version="1.0">
<description>Generates an empty flowfile, populates the contents with an example log HTTP request line, and extracts the HTTP response code. </description>
<groupId>b0049ce3-0157-1000-671f-21b7e01c3bcf</groupId>
<name>ExtractText Regex</name>
<snippet>
<connections>
<id>b0057ecb-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>b0049ce3-0157-1000-0000-000000000000</groupId>
<id>b004d6f1-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>b0049ce3-0157-1000-0000-000000000000</groupId>
<id>b004bdf9-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>b005f497-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>b0049ce3-0157-1000-0000-000000000000</groupId>
<id>b004fc6e-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>b0049ce3-0157-1000-0000-000000000000</groupId>
<id>b004d6f1-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>b006bb41-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>b0049ce3-0157-1000-0000-000000000000</groupId>
<id>b0051950-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>matched</selectedRelationships>
<selectedRelationships>unmatched</selectedRelationships>
<source>
<groupId>b0049ce3-0157-1000-0000-000000000000</groupId>
<id>b004fc6e-0157-1000-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<processors>
<id>b004bdf9-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>File Size</key>
<value>
<name>File Size</name>
</value>
</entry>
<entry>
<key>Batch Size</key>
<value>
<name>Batch Size</name>
</value>
</entry>
<entry>
<key>Data Format</key>
<value>
<name>Data Format</name>
</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>
<name>Unique FlowFiles</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>File Size</key>
<value>0B</value>
</entry>
<entry>
<key>Batch Size</key>
<value>1</value>
</entry>
<entry>
<key>Data Format</key>
<value>Text</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>false</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>3 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>GenerateFlowFile</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.GenerateFlowFile</type>
</processors>
<processors>
<id>b004d6f1-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>4.0</x>
<y>163.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Regular Expression</key>
<value>
<name>Regular Expression</name>
</value>
</entry>
<entry>
<key>Replacement Value</key>
<value>
<name>Replacement Value</name>
</value>
</entry>
<entry>
<key>Character Set</key>
<value>
<name>Character Set</name>
</value>
</entry>
<entry>
<key>Maximum Buffer Size</key>
<value>
<name>Maximum Buffer Size</name>
</value>
</entry>
<entry>
<key>Replacement Strategy</key>
<value>
<name>Replacement Strategy</name>
</value>
</entry>
<entry>
<key>Evaluation Mode</key>
<value>
<name>Evaluation Mode</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Regular Expression</key>
<value>(?s)(^.*$)</value>
</entry>
<entry>
<key>Replacement Value</key>
<value>133.43.96.45 - - [01/Aug/1995:00:00:16 -0400] "GET /shuttle/missions/sts-69/mission-sts-69.html HTTP/1.0" 200 10566</value>
</entry>
<entry>
<key>Character Set</key>
<value>UTF-8</value>
</entry>
<entry>
<key>Maximum Buffer Size</key>
<value>1 MB</value>
</entry>
<entry>
<key>Replacement Strategy</key>
<value>Regex Replace</value>
</entry>
<entry>
<key>Evaluation Mode</key>
<value>Entire text</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ReplaceText</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.ReplaceText</type>
</processors>
<processors>
<id>b004fc6e-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>7.0</x>
<y>347.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Character Set</key>
<value>
<name>Character Set</name>
</value>
</entry>
<entry>
<key>Maximum Buffer Size</key>
<value>
<name>Maximum Buffer Size</name>
</value>
</entry>
<entry>
<key>Maximum Capture Group Length</key>
<value>
<name>Maximum Capture Group Length</name>
</value>
</entry>
<entry>
<key>Enable Canonical Equivalence</key>
<value>
<name>Enable Canonical Equivalence</name>
</value>
</entry>
<entry>
<key>Enable Case-insensitive Matching</key>
<value>
<name>Enable Case-insensitive Matching</name>
</value>
</entry>
<entry>
<key>Permit Whitespace and Comments in Pattern</key>
<value>
<name>Permit Whitespace and Comments in Pattern</name>
</value>
</entry>
<entry>
<key>Enable DOTALL Mode</key>
<value>
<name>Enable DOTALL Mode</name>
</value>
</entry>
<entry>
<key>Enable Literal Parsing of the Pattern</key>
<value>
<name>Enable Literal Parsing of the Pattern</name>
</value>
</entry>
<entry>
<key>Enable Multiline Mode</key>
<value>
<name>Enable Multiline Mode</name>
</value>
</entry>
<entry>
<key>Enable Unicode-aware Case Folding</key>
<value>
<name>Enable Unicode-aware Case Folding</name>
</value>
</entry>
<entry>
<key>Enable Unicode Predefined Character Classes</key>
<value>
<name>Enable Unicode Predefined Character Classes</name>
</value>
</entry>
<entry>
<key>Enable Unix Lines Mode</key>
<value>
<name>Enable Unix Lines Mode</name>
</value>
</entry>
<entry>
<key>Include Capture Group 0</key>
<value>
<name>Include Capture Group 0</name>
</value>
</entry>
<entry>
<key>HTTP response</key>
<value>
<name>HTTP response</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Character Set</key>
<value>UTF-8</value>
</entry>
<entry>
<key>Maximum Buffer Size</key>
<value>1 MB</value>
</entry>
<entry>
<key>Maximum Capture Group Length</key>
<value>1024</value>
</entry>
<entry>
<key>Enable Canonical Equivalence</key>
<value>false</value>
</entry>
<entry>
<key>Enable Case-insensitive Matching</key>
<value>false</value>
</entry>
<entry>
<key>Permit Whitespace and Comments in Pattern</key>
<value>false</value>
</entry>
<entry>
<key>Enable DOTALL Mode</key>
<value>false</value>
</entry>
<entry>
<key>Enable Literal Parsing of the Pattern</key>
<value>false</value>
</entry>
<entry>
<key>Enable Multiline Mode</key>
<value>false</value>
</entry>
<entry>
<key>Enable Unicode-aware Case Folding</key>
<value>false</value>
</entry>
<entry>
<key>Enable Unicode Predefined Character Classes</key>
<value>false</value>
</entry>
<entry>
<key>Enable Unix Lines Mode</key>
<value>false</value>
</entry>
<entry>
<key>Include Capture Group 0</key>
<value>true</value>
</entry>
<entry>
<key>HTTP response</key>
<value>HTTP\/\d\.\d" (\d{3})</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ExtractText</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>matched</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>unmatched</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.ExtractText</type>
</processors>
<processors>
<id>b0051950-0157-1000-0000-000000000000</id>
<parentGroupId>b0049ce3-0157-1000-0000-000000000000</parentGroupId>
<position>
<x>6.0</x>
<y>523.0</y>
</position>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Log Level</key>
<value>
<name>Log Level</name>
</value>
</entry>
<entry>
<key>Log Payload</key>
<value>
<name>Log Payload</name>
</value>
</entry>
<entry>
<key>Attributes to Log</key>
<value>
<name>Attributes to Log</name>
</value>
</entry>
<entry>
<key>Attributes to Ignore</key>
<value>
<name>Attributes to Ignore</name>
</value>
</entry>
<entry>
<key>Log prefix</key>
<value>
<name>Log prefix</name>
</value>
</entry>
</descriptors>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Log Level</key>
<value>info</value>
</entry>
<entry>
<key>Log Payload</key>
<value>true</value>
</entry>
<entry>
<key>Attributes to Log</key>
</entry>
<entry>
<key>Attributes to Ignore</key>
</entry>
<entry>
<key>Log prefix</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>LogAttribute</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.LogAttribute</type>
</processors>
</snippet>
<timestamp>10/10/2016 12:20:53 PDT</timestamp>
</template>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment