Skip to content

Instantly share code, notes, and snippets.

@YolandaMDavis

YolandaMDavis/GetTwitterData-MongoSQL.xml

Created May 21, 2017 20:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save YolandaMDavis/9638cb9a124320eff025bf34e7a04c98 to your computer and use it in GitHub Desktop.
Save YolandaMDavis/9638cb9a124320eff025bf34e7a04c98 to your computer and use it in GitHub Desktop.
Download ZIP
Managing Data Flow with NiFi - Demo Templates
Raw
GetTwitterData-MongoSQL.xml
<?xml version="1.0" ?>
<template encoding-version="1.1">
<description></description>
<groupId>142ee790-015c-1000-9a0d-a31aac7e0eb3</groupId>
<name>GetTwitterDataSQLMongo-2017</name>
<snippet>
<connections>
<id>bcbbda47-e17a-32fb-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>95e97293-a576-3592-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>c7356249-dc89-39ee-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>56381546-0b2e-3924-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>e998c447-b463-3fe5-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>04389c9f-7be7-3f7d-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>sql</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>613ed4cc-8e2a-32d3-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>2c8b71b3-9ad0-358e-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>f7e5f138-c92c-37ab-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>3180b373-66b5-3254-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>613ed4cc-8e2a-32d3-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<controllerServices>
<id>2b06aa19-db49-3da9-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<bundle>
<artifact>nifi-dbcp-service-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<comments></comments>
<descriptors>
<entry>
<key>Database Connection URL</key>
<value>
<name>Database Connection URL</name>
</value>
</entry>
<entry>
<key>Database Driver Class Name</key>
<value>
<name>Database Driver Class Name</name>
</value>
</entry>
<entry>
<key>database-driver-locations</key>
<value>
<name>database-driver-locations</name>
</value>
</entry>
<entry>
<key>Database User</key>
<value>
<name>Database User</name>
</value>
</entry>
<entry>
<key>Password</key>
<value>
<name>Password</name>
</value>
</entry>
<entry>
<key>Max Wait Time</key>
<value>
<name>Max Wait Time</name>
</value>
</entry>
<entry>
<key>Max Total Connections</key>
<value>
<name>Max Total Connections</name>
</value>
</entry>
<entry>
<key>Validation-query</key>
<value>
<name>Validation-query</name>
</value>
</entry>
</descriptors>
<name>DBCPConnectionPool</name>
<persistsState>false</persistsState>
<properties>
<entry>
<key>Database Connection URL</key>
<value>jdbc:mysql://localhost:3306/twitter_dump</value>
</entry>
<entry>
<key>Database Driver Class Name</key>
<value>com.mysql.jdbc.Driver</value>
</entry>
<entry>
<key>database-driver-locations</key>
<value>/Users/ydavis/dev/tools/mysql-connector-java-5.1.42/mysql-connector-java-5.1.42-bin.jar</value>
</entry>
<entry>
<key>Database User</key>
<value>devanalyst</value>
</entry>
<entry>
<key>Password</key>
</entry>
<entry>
<key>Max Wait Time</key>
<value>500 millis</value>
</entry>
<entry>
<key>Max Total Connections</key>
<value>8</value>
</entry>
<entry>
<key>Validation-query</key>
</entry>
</properties>
<state>ENABLED</state>
<type>org.apache.nifi.dbcp.DBCPConnectionPool</type>
</controllerServices>
<processors>
<id>95e97293-a576-3592-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>245.19835042953494</y>
</position>
<bundle>
<artifact>nifi-social-media-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Twitter Endpoint</key>
<value>
<name>Twitter Endpoint</name>
</value>
</entry>
<entry>
<key>Consumer Key</key>
<value>
<name>Consumer Key</name>
</value>
</entry>
<entry>
<key>Consumer Secret</key>
<value>
<name>Consumer Secret</name>
</value>
</entry>
<entry>
<key>Access Token</key>
<value>
<name>Access Token</name>
</value>
</entry>
<entry>
<key>Access Token Secret</key>
<value>
<name>Access Token Secret</name>
</value>
</entry>
<entry>
<key>Languages</key>
<value>
<name>Languages</name>
</value>
</entry>
<entry>
<key>Terms to Filter On</key>
<value>
<name>Terms to Filter On</name>
</value>
</entry>
<entry>
<key>IDs to Follow</key>
<value>
<name>IDs to Follow</name>
</value>
</entry>
<entry>
<key>Locations to Filter On</key>
<value>
<name>Locations to Filter On</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Twitter Endpoint</key>
<value>Filter Endpoint</value>
</entry>
<entry>
<key>Consumer Key</key>
<value>PWGWQqIIlnMw1mVtlLqKap2tc</value>
</entry>
<entry>
<key>Consumer Secret</key>
</entry>
<entry>
<key>Access Token</key>
<value>548586960-SKaZmb67v7R33Nh6sz05BqaYVilRvXpBFvYUDOu5</value>
</entry>
<entry>
<key>Access Token Secret</key>
</entry>
<entry>
<key>Languages</key>
</entry>
<entry>
<key>Terms to Filter On</key>
<value>Baltimore</value>
</entry>
<entry>
<key>IDs to Follow</key>
</entry>
<entry>
<key>Locations to Filter On</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>5 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>GetTwitter</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.twitter.GetTwitter</type>
</processors>
<processors>
<id>f7e5f138-c92c-37ab-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>648.9961853027344</x>
<y>683.5735080488874</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Directory</key>
<value>
<name>Directory</name>
</value>
</entry>
<entry>
<key>Conflict Resolution Strategy</key>
<value>
<name>Conflict Resolution Strategy</name>
</value>
</entry>
<entry>
<key>Create Missing Directories</key>
<value>
<name>Create Missing Directories</name>
</value>
</entry>
<entry>
<key>Maximum File Count</key>
<value>
<name>Maximum File Count</name>
</value>
</entry>
<entry>
<key>Last Modified Time</key>
<value>
<name>Last Modified Time</name>
</value>
</entry>
<entry>
<key>Permissions</key>
<value>
<name>Permissions</name>
</value>
</entry>
<entry>
<key>Owner</key>
<value>
<name>Owner</name>
</value>
</entry>
<entry>
<key>Group</key>
<value>
<name>Group</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Directory</key>
<value>/Users/ydavis/Documents/Work/Hortonworks/HDF/Customer Presentations/Capital One/twitter-demo</value>
</entry>
<entry>
<key>Conflict Resolution Strategy</key>
<value>fail</value>
</entry>
<entry>
<key>Create Missing Directories</key>
<value>true</value>
</entry>
<entry>
<key>Maximum File Count</key>
</entry>
<entry>
<key>Last Modified Time</key>
</entry>
<entry>
<key>Permissions</key>
</entry>
<entry>
<key>Owner</key>
</entry>
<entry>
<key>Group</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>PutFile</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.PutFile</type>
</processors>
<processors>
<id>04389c9f-7be7-3f7d-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>1318.0425720214844</x>
<y>679.7179168041337</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>JDBC Connection Pool</key>
<value>
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService>
<name>JDBC Connection Pool</name>
</value>
</entry>
<entry>
<key>Support Fragmented Transactions</key>
<value>
<name>Support Fragmented Transactions</name>
</value>
</entry>
<entry>
<key>Transaction Timeout</key>
<value>
<name>Transaction Timeout</name>
</value>
</entry>
<entry>
<key>Batch Size</key>
<value>
<name>Batch Size</name>
</value>
</entry>
<entry>
<key>Obtain Generated Keys</key>
<value>
<name>Obtain Generated Keys</name>
</value>
</entry>
<entry>
<key>rollback-on-failure</key>
<value>
<name>rollback-on-failure</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>JDBC Connection Pool</key>
<value>2b06aa19-db49-3da9-0000-000000000000</value>
</entry>
<entry>
<key>Support Fragmented Transactions</key>
<value>true</value>
</entry>
<entry>
<key>Transaction Timeout</key>
</entry>
<entry>
<key>Batch Size</key>
<value>100</value>
</entry>
<entry>
<key>Obtain Generated Keys</key>
<value>false</value>
</entry>
<entry>
<key>rollback-on-failure</key>
<value>false</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>PutSQL</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>retry</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.PutSQL</type>
</processors>
<processors>
<id>56381546-0b2e-3924-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>1301.1581115722656</x>
<y>0.0</y>
</position>
<bundle>
<artifact>nifi-mongodb-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Mongo URI</key>
<value>
<name>Mongo URI</name>
</value>
</entry>
<entry>
<key>Mongo Database Name</key>
<value>
<name>Mongo Database Name</name>
</value>
</entry>
<entry>
<key>Mongo Collection Name</key>
<value>
<name>Mongo Collection Name</name>
</value>
</entry>
<entry>
<key>ssl-context-service</key>
<value>
<identifiesControllerService>org.apache.nifi.ssl.SSLContextService</identifiesControllerService>
<name>ssl-context-service</name>
</value>
</entry>
<entry>
<key>ssl-client-auth</key>
<value>
<name>ssl-client-auth</name>
</value>
</entry>
<entry>
<key>Mode</key>
<value>
<name>Mode</name>
</value>
</entry>
<entry>
<key>Upsert</key>
<value>
<name>Upsert</name>
</value>
</entry>
<entry>
<key>Update Query Key</key>
<value>
<name>Update Query Key</name>
</value>
</entry>
<entry>
<key>Write Concern</key>
<value>
<name>Write Concern</name>
</value>
</entry>
<entry>
<key>Character Set</key>
<value>
<name>Character Set</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Mongo URI</key>
<value>mongodb://localhost:27017</value>
</entry>
<entry>
<key>Mongo Database Name</key>
<value>twitter_dump</value>
</entry>
<entry>
<key>Mongo Collection Name</key>
<value>subset-tweets</value>
</entry>
<entry>
<key>ssl-context-service</key>
</entry>
<entry>
<key>ssl-client-auth</key>
<value>REQUIRED</value>
</entry>
<entry>
<key>Mode</key>
<value>insert</value>
</entry>
<entry>
<key>Upsert</key>
<value>false</value>
</entry>
<entry>
<key>Update Query Key</key>
<value>_id</value>
</entry>
<entry>
<key>Write Concern</key>
<value>ACKNOWLEDGED</value>
</entry>
<entry>
<key>Character Set</key>
<value>UTF-8</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>PutMongo</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.mongodb.PutMongo</type>
</processors>
<processors>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>643.7197265625</x>
<y>251.77098941802979</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>jolt-transform</key>
<value>
<name>jolt-transform</name>
</value>
</entry>
<entry>
<key>jolt-custom-class</key>
<value>
<name>jolt-custom-class</name>
</value>
</entry>
<entry>
<key>jolt-custom-modules</key>
<value>
<name>jolt-custom-modules</name>
</value>
</entry>
<entry>
<key>jolt-spec</key>
<value>
<name>jolt-spec</name>
</value>
</entry>
<entry>
<key>Transform Cache Size</key>
<value>
<name>Transform Cache Size</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>jolt-transform</key>
<value>jolt-transform-chain</value>
</entry>
<entry>
<key>jolt-custom-class</key>
</entry>
<entry>
<key>jolt-custom-modules</key>
</entry>
<entry>
<key>jolt-spec</key>
<value>[{
"operation": "shift",
"spec": {
"id": "tweet_id",
"text": "tweet_text"
}
},{
"operation": "modify-default-beta",
"spec": {
"~in_reply_to_status_id": 0,
"~in_reply_to_user_id": "",
"~in_reply_to_user_id_str": 0,
"~in_reply_to_screen_name": ""
}
}]</value>
</entry>
<entry>
<key>Transform Cache Size</key>
<value>1000</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>JoltTransformJSON</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.JoltTransformJSON</type>
</processors>
<processors>
<id>613ed4cc-8e2a-32d3-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>1315.9320983886719</x>
<y>260.43652663069304</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>JDBC Connection Pool</key>
<value>
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService>
<name>JDBC Connection Pool</name>
</value>
</entry>
<entry>
<key>Statement Type</key>
<value>
<name>Statement Type</name>
</value>
</entry>
<entry>
<key>Table Name</key>
<value>
<name>Table Name</name>
</value>
</entry>
<entry>
<key>Catalog Name</key>
<value>
<name>Catalog Name</name>
</value>
</entry>
<entry>
<key>Schema Name</key>
<value>
<name>Schema Name</name>
</value>
</entry>
<entry>
<key>Translate Field Names</key>
<value>
<name>Translate Field Names</name>
</value>
</entry>
<entry>
<key>Unmatched Field Behavior</key>
<value>
<name>Unmatched Field Behavior</name>
</value>
</entry>
<entry>
<key>Unmatched Column Behavior</key>
<value>
<name>Unmatched Column Behavior</name>
</value>
</entry>
<entry>
<key>Update Keys</key>
<value>
<name>Update Keys</name>
</value>
</entry>
<entry>
<key>jts-quoted-identifiers</key>
<value>
<name>jts-quoted-identifiers</name>
</value>
</entry>
<entry>
<key>jts-quoted-table-identifiers</key>
<value>
<name>jts-quoted-table-identifiers</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>JDBC Connection Pool</key>
<value>2b06aa19-db49-3da9-0000-000000000000</value>
</entry>
<entry>
<key>Statement Type</key>
<value>INSERT</value>
</entry>
<entry>
<key>Table Name</key>
<value>twitter_dump.`subset-tweets`</value>
</entry>
<entry>
<key>Catalog Name</key>
</entry>
<entry>
<key>Schema Name</key>
</entry>
<entry>
<key>Translate Field Names</key>
<value>true</value>
</entry>
<entry>
<key>Unmatched Field Behavior</key>
<value>Ignore Unmatched Fields</value>
</entry>
<entry>
<key>Unmatched Column Behavior</key>
<value>Fail on Unmatched Columns</value>
</entry>
<entry>
<key>Update Keys</key>
</entry>
<entry>
<key>jts-quoted-identifiers</key>
<value>false</value>
</entry>
<entry>
<key>jts-quoted-table-identifiers</key>
<value>false</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>ConvertJSONToSQL</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>original</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>sql</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.ConvertJSONToSQL</type>
</processors>
</snippet>
<timestamp>05/21/2017 16:36:53 EDT</timestamp>
</template>
Raw
GetTwitterData.xml
<?xml version="1.0" ?>
<template encoding-version="1.1">
<description></description>
<groupId>142ee790-015c-1000-9a0d-a31aac7e0eb3</groupId>
<name>GetTwitterData-2017</name>
<snippet>
<connections>
<id>bcbbda47-e17a-32fb-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>95e97293-a576-3592-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>2c8b71b3-9ad0-358e-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>f7e5f138-c92c-37ab-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>e5a17068-5da9-3260-0000-000000000000</groupId>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<processors>
<id>95e97293-a576-3592-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<bundle>
<artifact>nifi-social-media-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Twitter Endpoint</key>
<value>
<name>Twitter Endpoint</name>
</value>
</entry>
<entry>
<key>Consumer Key</key>
<value>
<name>Consumer Key</name>
</value>
</entry>
<entry>
<key>Consumer Secret</key>
<value>
<name>Consumer Secret</name>
</value>
</entry>
<entry>
<key>Access Token</key>
<value>
<name>Access Token</name>
</value>
</entry>
<entry>
<key>Access Token Secret</key>
<value>
<name>Access Token Secret</name>
</value>
</entry>
<entry>
<key>Languages</key>
<value>
<name>Languages</name>
</value>
</entry>
<entry>
<key>Terms to Filter On</key>
<value>
<name>Terms to Filter On</name>
</value>
</entry>
<entry>
<key>IDs to Follow</key>
<value>
<name>IDs to Follow</name>
</value>
</entry>
<entry>
<key>Locations to Filter On</key>
<value>
<name>Locations to Filter On</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Twitter Endpoint</key>
<value>Filter Endpoint</value>
</entry>
<entry>
<key>Consumer Key</key>
<value>PWGWQqIIlnMw1mVtlLqKap2tc</value>
</entry>
<entry>
<key>Consumer Secret</key>
</entry>
<entry>
<key>Access Token</key>
<value>548586960-SKaZmb67v7R33Nh6sz05BqaYVilRvXpBFvYUDOu5</value>
</entry>
<entry>
<key>Access Token Secret</key>
</entry>
<entry>
<key>Languages</key>
</entry>
<entry>
<key>Terms to Filter On</key>
<value>Baltimore</value>
</entry>
<entry>
<key>IDs to Follow</key>
</entry>
<entry>
<key>Locations to Filter On</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>5 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>GetTwitter</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.twitter.GetTwitter</type>
</processors>
<processors>
<id>f7e5f138-c92c-37ab-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>648.9961853027344</x>
<y>438.37515761935254</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Directory</key>
<value>
<name>Directory</name>
</value>
</entry>
<entry>
<key>Conflict Resolution Strategy</key>
<value>
<name>Conflict Resolution Strategy</name>
</value>
</entry>
<entry>
<key>Create Missing Directories</key>
<value>
<name>Create Missing Directories</name>
</value>
</entry>
<entry>
<key>Maximum File Count</key>
<value>
<name>Maximum File Count</name>
</value>
</entry>
<entry>
<key>Last Modified Time</key>
<value>
<name>Last Modified Time</name>
</value>
</entry>
<entry>
<key>Permissions</key>
<value>
<name>Permissions</name>
</value>
</entry>
<entry>
<key>Owner</key>
<value>
<name>Owner</name>
</value>
</entry>
<entry>
<key>Group</key>
<value>
<name>Group</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Directory</key>
<value>/Users/ydavis/Documents/Work/Hortonworks/HDF/Customer Presentations/Capital One/twitter-demo</value>
</entry>
<entry>
<key>Conflict Resolution Strategy</key>
<value>fail</value>
</entry>
<entry>
<key>Create Missing Directories</key>
<value>true</value>
</entry>
<entry>
<key>Maximum File Count</key>
</entry>
<entry>
<key>Last Modified Time</key>
</entry>
<entry>
<key>Permissions</key>
</entry>
<entry>
<key>Owner</key>
</entry>
<entry>
<key>Group</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>PutFile</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.PutFile</type>
</processors>
<processors>
<id>5e74cd11-b091-337a-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<position>
<x>643.7197265625</x>
<y>6.572638988494845</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.3.0-SNAPSHOT</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>jolt-transform</key>
<value>
<name>jolt-transform</name>
</value>
</entry>
<entry>
<key>jolt-custom-class</key>
<value>
<name>jolt-custom-class</name>
</value>
</entry>
<entry>
<key>jolt-custom-modules</key>
<value>
<name>jolt-custom-modules</name>
</value>
</entry>
<entry>
<key>jolt-spec</key>
<value>
<name>jolt-spec</name>
</value>
</entry>
<entry>
<key>Transform Cache Size</key>
<value>
<name>Transform Cache Size</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>jolt-transform</key>
<value>jolt-transform-chain</value>
</entry>
<entry>
<key>jolt-custom-class</key>
</entry>
<entry>
<key>jolt-custom-modules</key>
</entry>
<entry>
<key>jolt-spec</key>
<value>[{
"operation": "shift",
"spec": {
"id": "tweet_id",
"text": "tweet_text"
}
},{
"operation": "modify-default-beta",
"spec": {
"~in_reply_to_status_id": 0,
"~in_reply_to_user_id": "",
"~in_reply_to_user_id_str": 0,
"~in_reply_to_screen_name": ""
}
}]</value>
</entry>
<entry>
<key>Transform Cache Size</key>
<value>1000</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<name>JoltTransformJSON</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<style></style>
<type>org.apache.nifi.processors.standard.JoltTransformJSON</type>
</processors>
</snippet>
<timestamp>05/21/2017 16:35:57 EDT</timestamp>
</template>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment