Skip to content

Instantly share code, notes, and snippets.

@Kulasangar

Kulasangar/flow.xml

Last active January 15, 2022 19:48
Show Gist options
  • Save Kulasangar/e604ded2f950ffe2d31f6846d559d0cb to your computer and use it in GitHub Desktop.
Save Kulasangar/e604ded2f950ffe2d31f6846d559d0cb to your computer and use it in GitHub Desktop.
Download ZIP
Replace your flowfile with this and test
Raw
flow.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<flowController encoding-version="1.4">
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>1</maxEventDrivenThreadCount>
<registries/>
<parameterContexts/>
<rootGroup>
<id>2cf1e375-017d-1000-1dc6-f43e1ff015a7</id>
<name>NiFi Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<defaultFlowFileExpiration>0 sec</defaultFlowFileExpiration>
<defaultBackPressureObjectThreshold>10000</defaultBackPressureObjectThreshold>
<defaultBackPressureDataSizeThreshold>1 GB</defaultBackPressureDataSizeThreshold>
<processor>
<id>017d1000-60d8-11da-2efa-dce7047b6bbe</id>
<name>PutElasticsearchHttp</name>
<position x="2840.0" y="320.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.elasticsearch.PutElasticsearchHttp</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-elasticsearch-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>elasticsearch-http-url</name>
<value>http://127.0.0.1:9200</value>
</property>
<property>
<name>SSL Context Service</name>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>Username</name>
</property>
<property>
<name>Password</name>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
<value>5 secs</value>
</property>
<property>
<name>elasticsearch-http-response-timeout</name>
<value>15 secs</value>
</property>
<property>
<name>proxy-configuration-service</name>
</property>
<property>
<name>elasticsearch-http-proxy-host</name>
</property>
<property>
<name>elasticsearch-http-proxy-port</name>
</property>
<property>
<name>proxy-username</name>
</property>
<property>
<name>proxy-password</name>
</property>
<property>
<name>put-es-id-attr</name>
</property>
<property>
<name>put-es-index</name>
<value>twitter_feed</value>
</property>
<property>
<name>put-es-type</name>
<value>_doc</value>
</property>
<property>
<name>put-es-batch-size</name>
<value>100</value>
</property>
<property>
<name>put-es-index-op</name>
<value>index</value>
</property>
<autoTerminatedRelationship>success</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>5207b2a9-017d-1000-fdc3-513c318a9ef9</id>
<name>SplitJson</name>
<position x="2840.0" y="120.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.standard.SplitJson</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>JsonPath Expression</name>
<value>$</value>
</property>
<property>
<name>Null Value Representation</name>
<value>empty string</value>
</property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
<autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
<processor>
<id>4affcf14-017d-1000-061c-ace28f6513d0</id>
<name>JoltTransformJSON</name>
<position x="1720.0" y="312.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.standard.JoltTransformJSON</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>jolt-transform</name>
<value>jolt-transform-chain</value>
</property>
<property>
<name>jolt-custom-class</name>
</property>
<property>
<name>jolt-custom-modules</name>
</property>
<property>
<name>jolt-spec</name>
<value>[{
"operation": "shift",
"spec": {
"id": "tweet_id",
"created_at": "created_date",
"text": "text",
"coordinates": "coordinates",
"user": {
"id": "user_id",
"name": "name",
"screen_name": "screen_name",
"location": "location"
}
}
}]</value>
</property>
<property>
<name>Transform Cache Size</name>
<value>1</value>
</property>
<property>
<name>pretty_print</name>
<value>false</value>
</property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>2d2c0b2a-017d-1000-23a7-9a2f8f708f1c</id>
<name>GetTwitter</name>
<position x="1112.0" y="312.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.twitter.GetTwitter</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-social-media-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Twitter Endpoint</name>
<value>Filter Endpoint</value>
</property>
<property>
<name>max-client-error-retries</name>
<value>5</value>
</property>
<property>
<name>Consumer Key</name>
<value>enc{eb1fcc5b3665c67c88f740cfd046911e48b6cada15694c8be87cc00249e4656d42fb8710d254f6c6c00ca3401076086890578dd6f0243b75e2}</value>
</property>
<property>
<name>Consumer Secret</name>
<value>enc{96fef6d2eb56035031adf821999cf96c0eb6486bef87d07118152265643d26d08a9a1e379a1b072d2dcc05fc70e9be3b27c2c0066bae10b3c323e8d7b219f35f2a697bff0fe0ff554a740689f59a8b7369b3}</value>
</property>
<property>
<name>Access Token</name>
<value>enc{6c31a28df097591373abd6fa0c7b725e281bf79fbef943de41c3e13145c710222ae89eeaf73da63f26bba120687951c00a3825b1fcb33af75bdcbb1b2e85e1d67c28ec0fa56fb2d5f6fc931f72e6b1e721f4}</value>
</property>
<property>
<name>Access Token Secret</name>
<value>enc{264761deadb72113bbbb3b88a76a83456efa3c1148f0f59b97192c7cdf7e72d77660a769f306eb1425291f89ce8c7e37fadc2760987d24b247e235a81eef58012406c551bba3687db3b518a8ef}</value>
</property>
<property>
<name>Languages</name>
<value>en</value>
</property>
<property>
<name>Terms to Filter On</name>
<value>covid, covid-19, Covid, Covid-19, vaccine, covid cases</value>
</property>
<property>
<name>IDs to Follow</name>
</property>
<property>
<name>Locations to Filter On</name>
</property>
</processor>
<processor>
<id>017d1032-60d8-11da-1b3b-d33f0beb95df</id>
<name>PublishKafkaRecord_2_0</name>
<position x="2088.0" y="-152.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-kafka-2-0-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>bootstrap.servers</name>
<value>localhost:9092</value>
</property>
<property>
<name>topic</name>
<value>twitterfeed</value>
</property>
<property>
<name>record-reader</name>
<value>017d1033-60d8-11da-9e43-40c8c7b56b73</value>
</property>
<property>
<name>record-writer</name>
<value>017d1034-60d8-11da-6b47-a7f8947ed101</value>
</property>
<property>
<name>use-transactions</name>
<value>true</value>
</property>
<property>
<name>Failure Strategy</name>
<value>Route to Failure</value>
</property>
<property>
<name>transactional-id-prefix</name>
</property>
<property>
<name>acks</name>
<value>all</value>
</property>
<property>
<name>attribute-name-regex</name>
</property>
<property>
<name>message-header-encoding</name>
<value>UTF-8</value>
</property>
<property>
<name>security.protocol</name>
<value>PLAINTEXT</value>
</property>
<property>
<name>sasl.mechanism</name>
<value>GSSAPI</value>
</property>
<property>
<name>kerberos-credentials-service</name>
</property>
<property>
<name>sasl.kerberos.service.name</name>
</property>
<property>
<name>sasl.kerberos.principal</name>
</property>
<property>
<name>sasl.kerberos.keytab</name>
</property>
<property>
<name>sasl.username</name>
</property>
<property>
<name>sasl.password</name>
</property>
<property>
<name>sasl.token.auth</name>
<value>false</value>
</property>
<property>
<name>ssl.context.service</name>
</property>
<property>
<name>message-key-field</name>
</property>
<property>
<name>max.request.size</name>
<value>1 MB</value>
</property>
<property>
<name>ack.wait.time</name>
<value>5 secs</value>
</property>
<property>
<name>max.block.ms</name>
<value>5 sec</value>
</property>
<property>
<name>partitioner.class</name>
<value>org.apache.kafka.clients.producer.internals.DefaultPartitioner</value>
</property>
<property>
<name>partition</name>
</property>
<property>
<name>compression.type</name>
<value>none</value>
</property>
<autoTerminatedRelationship>success</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>4e1b997f-017d-1000-6cc4-1a15f10088e3</id>
<name>ConvertJSONToSQL</name>
<position x="2240.0" y="496.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.standard.ConvertJSONToSQL</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>JDBC Connection Pool</name>
<value>4e1c2239-017d-1000-3163-21d6b012480d</value>
</property>
<property>
<name>Statement Type</name>
<value>INSERT</value>
</property>
<property>
<name>Table Name</name>
<value>twitter_feed</value>
</property>
<property>
<name>Catalog Name</name>
</property>
<property>
<name>Schema Name</name>
</property>
<property>
<name>Translate Field Names</name>
<value>true</value>
</property>
<property>
<name>Unmatched Field Behavior</name>
<value>Ignore Unmatched Fields</value>
</property>
<property>
<name>Unmatched Column Behavior</name>
<value>Fail on Unmatched Columns</value>
</property>
<property>
<name>Update Keys</name>
</property>
<property>
<name>jts-quoted-identifiers</name>
<value>false</value>
</property>
<property>
<name>jts-quoted-table-identifiers</name>
<value>false</value>
</property>
<property>
<name>jts-sql-param-attr-prefix</name>
<value>sql</value>
</property>
<property>
<name>table-schema-cache-size</name>
<value>100</value>
</property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
<autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
<processor>
<id>017d1030-60d8-11da-7dab-372c9cdb774d</id>
<name>AttributeRollingWindow</name>
<position x="1480.0" y="-152.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.stateful.analysis.AttributeRollingWindow</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-stateful-analysis-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Value to track</name>
<value>5</value>
</property>
<property>
<name>Time window</name>
<value>30s</value>
</property>
<property>
<name>Sub-window length</name>
</property>
<autoTerminatedRelationship>set state fail</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>51f4b158-017d-1000-3bdd-bc6d1a35352a</id>
<name>QueryRecord</name>
<position x="2240.0" y="120.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.standard.QueryRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>51f4fdf5-017d-1000-ae5e-2b99e8157fd2</value>
</property>
<property>
<name>record-writer</name>
<value>51f52c8c-017d-1000-40ea-87166f47c867</value>
</property>
<property>
<name>include-zero-record-flowfiles</name>
<value>false</value>
</property>
<property>
<name>cache-schema</name>
<value>true</value>
</property>
<property>
<name>dbf-default-precision</name>
<value>10</value>
</property>
<property>
<name>dbf-default-scale</name>
<value>0</value>
</property>
<property>
<name>query</name>
<value>select * from FLOWFILE</value>
</property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
<autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
<processor>
<id>4e450d8a-017d-1000-ea88-430fd7dfe114</id>
<name>PutSQL</name>
<position x="2848.0" y="488.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.standard.PutSQL</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>JDBC Connection Pool</name>
<value>4e1c2239-017d-1000-3163-21d6b012480d</value>
</property>
<property>
<name>putsql-sql-statement</name>
</property>
<property>
<name>Support Fragmented Transactions</name>
<value>true</value>
</property>
<property>
<name>database-session-autocommit</name>
<value>false</value>
</property>
<property>
<name>Transaction Timeout</name>
</property>
<property>
<name>Batch Size</name>
<value>100</value>
</property>
<property>
<name>Obtain Generated Keys</name>
<value>false</value>
</property>
<property>
<name>rollback-on-failure</name>
<value>false</value>
</property>
<autoTerminatedRelationship>success</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>017d1029-60d8-11da-9c8a-79e4dd432c22</id>
<name>PutS3Object</name>
<position x="1480.0" y="728.0"/>
<styles/>
<comment/>
<class>org.apache.nifi.processors.aws.s3.PutS3Object</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-aws-nar</artifact>
<version>1.15.0</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Object Key</name>
<value>${filename}</value>
</property>
<property>
<name>Bucket</name>
<value>wiley-datalake</value>
</property>
<property>
<name>Content Type</name>
</property>
<property>
<name>Content Disposition</name>
</property>
<property>
<name>Cache Control</name>
</property>
<property>
<name>Access Key</name>
<value>enc{dc864efabc82bc82e3e6be84f617efd44974beb7d380111226008b8e545bf31769a33f2fc19532ee89e9874485a7ef09243f6da2}</value>
</property>
<property>
<name>Secret Key</name>
<value>enc{5c0325d90aa5745e926a35606e2918dd9bacda9e200426e51b3ac70aceef810f461257d603c383feac8b49914af66ef4be78be7a6fc7539cf539d7be85ddbda99ed3f5f93287d51d}</value>
</property>
<property>
<name>Credentials File</name>
</property>
<property>
<name>AWS Credentials Provider service</name>
</property>
<property>
<name>s3-object-tags-prefix</name>
</property>
<property>
<name>s3-object-remove-tags-prefix</name>
<value>false</value>
</property>
<property>
<name>Storage Class</name>
<value>Standard</value>
</property>
<property>
<name>Region</name>
<value>us-east-1</value>
</property>
<property>
<name>Communications Timeout</name>
<value>30 secs</value>
</property>
<property>
<name>Expiration Time Rule</name>
</property>
<property>
<name>FullControl User List</name>
<value>${s3.permissions.full.users}</value>
</property>
<property>
<name>Read Permission User List</name>
<value>${s3.permissions.read.users}</value>
</property>
<property>
<name>Write Permission User List</name>
<value>${s3.permissions.write.users}</value>
</property>
<property>
<name>Read ACL User List</name>
<value>${s3.permissions.readacl.users}</value>
</property>
<property>
<name>Write ACL User List</name>
<value>${s3.permissions.writeacl.users}</value>
</property>
<property>
<name>Owner</name>
<value>${s3.owner}</value>
</property>
<property>
<name>canned-acl</name>
<value>${s3.permissions.cannedacl}</value>
</property>
<property>
<name>SSL Context Service</name>
</property>
<property>
<name>Endpoint Override URL</name>
</property>
<property>
<name>Signer Override</name>
<value>Default Signature</value>
</property>
<property>
<name>Multipart Threshold</name>
<value>5 GB</value>
</property>
<property>
<name>Multipart Part Size</name>
<value>5 GB</value>
</property>
<property>
<name>Multipart Upload AgeOff Interval</name>
<value>60 min</value>
</property>
<property>
<name>Multipart Upload Max Age Threshold</name>
<value>7 days</value>
</property>
<property>
<name>s3-temporary-directory-multipart</name>
<value>${java.io.tmpdir}</value>
</property>
<property>
<name>server-side-encryption</name>
<value>None</value>
</property>
<property>
<name>encryption-service</name>
</property>
<property>
<name>use-chunked-encoding</name>
<value>true</value>
</property>
<property>
<name>use-path-style-access</name>
<value>false</value>
</property>
<property>
<name>proxy-configuration-service</name>
</property>
<property>
<name>Proxy Host</name>
</property>
<property>
<name>Proxy Host Port</name>
</property>
<property>
<name>proxy-user-name</name>
</property>
<property>
<name>proxy-user-password</name>
</property>
<autoTerminatedRelationship>success</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<connection>
<id>4e451f1e-017d-1000-d3c9-dd9b2242fa89</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>4e1b997f-017d-1000-6cc4-1a15f10088e3</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>4e450d8a-017d-1000-ea88-430fd7dfe114</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>sql</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>017d1002-60d8-11da-52f6-a60a66493824</id>
<name/>
<bendPoints>
<bendPoint x="3311.1875" y="355.38800048828125"/>
<bendPoint x="3311.1875" y="405.38800048828125"/>
</bendPoints>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>017d1000-60d8-11da-2efa-dce7047b6bbe</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>017d1000-60d8-11da-2efa-dce7047b6bbe</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>retry</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>017d101b-60d8-11da-49d6-12b569b6f0e0</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>5207b2a9-017d-1000-fdc3-513c318a9ef9</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>017d1000-60d8-11da-2efa-dce7047b6bbe</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>split</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>017d1037-60d8-11da-f168-afc176e59dc2</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>017d1030-60d8-11da-7dab-372c9cdb774d</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>017d1032-60d8-11da-1b3b-d33f0beb95df</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>017d101a-60d8-11da-645b-32657a11d20b</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>51f4b158-017d-1000-3bdd-bc6d1a35352a</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>5207b2a9-017d-1000-fdc3-513c318a9ef9</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>query</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>4e44d9c0-017d-1000-15f9-54cb28700357</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>4affcf14-017d-1000-061c-ace28f6513d0</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>4e1b997f-017d-1000-6cc4-1a15f10088e3</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>4e47306b-017d-1000-1f3d-ed56aa134caa</id>
<name/>
<bendPoints>
<bendPoint x="3317.3734130859375" y="522.483642578125"/>
<bendPoint x="3317.3734130859375" y="572.483642578125"/>
</bendPoints>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>4e450d8a-017d-1000-ea88-430fd7dfe114</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>4e450d8a-017d-1000-ea88-430fd7dfe114</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>retry</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>4c8e3a54-017d-1000-a654-b615c4fb354e</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>2d2c0b2a-017d-1000-23a7-9a2f8f708f1c</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>4affcf14-017d-1000-061c-ace28f6513d0</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>520175d7-017d-1000-8bf2-87257cfffe90</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>4affcf14-017d-1000-061c-ace28f6513d0</sourceId>
<sourceGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>51f4b158-017d-1000-3bdd-bc6d1a35352a</destinationId>
<destinationGroupId>2cf1e375-017d-1000-1dc6-f43e1ff015a7</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute/>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<controllerService>
<id>4e1c2239-017d-1000-3163-21d6b012480d</id>
<name>DBCPConnectionPool</name>
<comment/>
<class>org.apache.nifi.dbcp.DBCPConnectionPool</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-dbcp-service-nar</artifact>
<version>1.15.0</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Database Connection URL</name>
<value>jdbc:mysql://127.0.0.1:3306/nifi_test</value>
</property>
<property>
<name>Database Driver Class Name</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<property>
<name>database-driver-locations</name>
<value>/mnt/c/Projects/hackathon-WIleySL/nifi-1.15.0-bin/nifi-1.15.0/jdbc_drivers/mysql-connector-java-8.0.22.jar</value>
</property>
<property>
<name>kerberos-user-service</name>
</property>
<property>
<name>kerberos-credentials-service</name>
</property>
<property>
<name>kerberos-principal</name>
</property>
<property>
<name>kerberos-password</name>
</property>
<property>
<name>Database User</name>
<value>root</value>
</property>
<property>
<name>Password</name>
<value>enc{f8032ca4bf1c602acb5096fe10accba83a9e0cf7ae074c2fd8bac0b5ba6370e8986c2dfe}</value>
</property>
<property>
<name>Max Wait Time</name>
<value>500 millis</value>
</property>
<property>
<name>Max Total Connections</name>
<value>8</value>
</property>
<property>
<name>Validation-query</name>
</property>
<property>
<name>dbcp-min-idle-conns</name>
<value>0</value>
</property>
<property>
<name>dbcp-max-idle-conns</name>
<value>8</value>
</property>
<property>
<name>dbcp-max-conn-lifetime</name>
<value>-1</value>
</property>
<property>
<name>dbcp-time-between-eviction-runs</name>
<value>-1</value>
</property>
<property>
<name>dbcp-min-evictable-idle-time</name>
<value>30 mins</value>
</property>
<property>
<name>dbcp-soft-min-evictable-idle-time</name>
<value>-1</value>
</property>
</controllerService>
<controllerService>
<id>51f4fdf5-017d-1000-ae5e-2b99e8157fd2</id>
<name>JsonTreeReader</name>
<comment/>
<class>org.apache.nifi.json.JsonTreeReader</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
<version>1.15.0</version>
</bundle>
<enabled>true</enabled>
<property>
<name>schema-access-strategy</name>
<value>schema-name</value>
</property>
<property>
<name>schema-registry</name>
<value>51f81664-017d-1000-98dd-ecea15a08759</value>
</property>
<property>
<name>schema-name</name>
<value>json.tweets</value>
</property>
<property>
<name>schema-version</name>
</property>
<property>
<name>schema-branch</name>
</property>
<property>
<name>schema-text</name>
<value>${avro.schema}</value>
</property>
<property>
<name>schema-inference-cache</name>
</property>
<property>
<name>Date Format</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
</property>
</controllerService>
<controllerService>
<id>017d1034-60d8-11da-6b47-a7f8947ed101</id>
<name>JsonRecordSetWriter</name>
<comment/>
<class>org.apache.nifi.json.JsonRecordSetWriter</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
<version>1.15.0</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Schema Write Strategy</name>
<value>no-schema</value>
</property>
<property>
<name>schema-cache</name>
</property>
<property>
<name>schema-protocol-version</name>
<value>1</value>
</property>
<property>
<name>schema-access-strategy</name>
<value>inherit-record-schema</value>
</property>
<property>
<name>schema-registry</name>
</property>
<property>
<name>schema-name</name>
<value>${schema.name}</value>
</property>
<property>
<name>schema-version</name>
</property>
<property>
<name>schema-branch</name>
</property>
<property>
<name>schema-text</name>
<value>${avro.schema}</value>
</property>
<property>
<name>Date Format</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
</property>
<property>
<name>Pretty Print JSON</name>
<value>false</value>
</property>
<property>
<name>suppress-nulls</name>
<value>never-suppress</value>
</property>
<property>
<name>output-grouping</name>
<value>output-array</value>
</property>
<property>
<name>compression-format</name>
<value>none</value>
</property>
<property>
<name>compression-level</name>
<value>1</value>
</property>
</controllerService>
<controllerService>
<id>017d1033-60d8-11da-9e43-40c8c7b56b73</id>
<name>JsonTreeReader</name>
<comment/>
<class>org.apache.nifi.json.JsonTreeReader</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
<version>1.15.0</version>
</bundle>
<enabled>true</enabled>
<property>
<name>schema-access-strategy</name>
<value>infer-schema</value>
</property>
<property>
<name>schema-registry</name>
</property>
<property>
<name>schema-name</name>
<value>${schema.name}</value>
</property>
<property>
<name>schema-version</name>
</property>
<property>
<name>schema-branch</name>
</property>
<property>
<name>schema-text</name>
<value>${avro.schema}</value>
</property>
<property>
<name>schema-inference-cache</name>
</property>
<property>
<name>Date Format</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
</property>
</controllerService>
<controllerService>
<id>51f52c8c-017d-1000-40ea-87166f47c867</id>
<name>JsonRecordSetWriter</name>
<comment/>
<class>org.apache.nifi.json.JsonRecordSetWriter</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
<version>1.15.0</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Schema Write Strategy</name>
<value>no-schema</value>
</property>
<property>
<name>schema-cache</name>
</property>
<property>
<name>schema-protocol-version</name>
<value>1</value>
</property>
<property>
<name>schema-access-strategy</name>
<value>inherit-record-schema</value>
</property>
<property>
<name>schema-registry</name>
</property>
<property>
<name>schema-name</name>
<value>${schema.name}</value>
</property>
<property>
<name>schema-version</name>
</property>
<property>
<name>schema-branch</name>
</property>
<property>
<name>schema-text</name>
<value>${avro.schema}</value>
</property>
<property>
<name>Date Format</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
</property>
<property>
<name>Pretty Print JSON</name>
<value>false</value>
</property>
<property>
<name>suppress-nulls</name>
<value>never-suppress</value>
</property>
<property>
<name>output-grouping</name>
<value>output-array</value>
</property>
<property>
<name>compression-format</name>
<value>none</value>
</property>
<property>
<name>compression-level</name>
<value>1</value>
</property>
</controllerService>
<controllerService>
<id>51f81664-017d-1000-98dd-ecea15a08759</id>
<name>AvroSchemaRegistry</name>
<comment/>
<class>org.apache.nifi.schemaregistry.services.AvroSchemaRegistry</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-registry-nar</artifact>
<version>1.15.0</version>
</bundle>
<enabled>true</enabled>
<property>
<name>avro-reg-validated-field-names</name>
<value>true</value>
</property>
<property>
<name>json.tweets</name>
<value>{
"type": "record",
"name":"jsonTweets",
"fields":[
{"name":"tweet_id", "type":["null", "string"]},
{"name":"created_date", "type":["null", "string"]},
{"name":"text", "type":["null", "string"]},
{"name":"user_id", "type":["null", "string"]},
{"name":"name", "type":["null", "string"]},
{"name":"screen_name", "type":["null", "string"]},
{"name":"location", "type":["null", "string"]}
]
}</value>
</property>
</controllerService>
</rootGroup>
<controllerServices/>
<reportingTasks/>
</flowController>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment