Created
June 13, 2017 02:40
-
-
Save ijokarumawak/0bd4e0572e776313e8d5056d7d04f444 to your computer and use it in GitHub Desktop.
NiFi example, using Avro logical type when copy MySQL records having decimal type
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0" ?> | |
<template encoding-version="1.1"> | |
<description></description> | |
<groupId>1e8778a8-015c-1000-c747-8273276bab45</groupId> | |
<name>MySQL Decimal</name> | |
<snippet> | |
<processGroups> | |
<id>fcff8588-6cae-364b-0000-000000000000</id> | |
<parentGroupId>5454b790-94fe-3e28-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.0</x> | |
<y>0.0</y> | |
</position> | |
<comments></comments> | |
<contents> | |
<connections> | |
<id>c7cc273d-5b73-3940-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>137b0f09-d80a-3c68-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>0</labelIndex> | |
<name></name> | |
<selectedRelationships>failure</selectedRelationships> | |
<source> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>60bec32b-990e-3aaf-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>c9ba4afc-77c3-306e-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>d870d851-d95d-3690-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>60bec32b-990e-3aaf-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>cfb5dac9-53cf-360a-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>137b0f09-d80a-3c68-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>d870d851-d95d-3690-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>ff0a120b-cf1c-39c7-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<bends> | |
<x>1075.0</x> | |
<y>913.0</y> | |
</bends> | |
<destination> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>137b0f09-d80a-3c68-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<name></name> | |
<selectedRelationships>failure</selectedRelationships> | |
<selectedRelationships>retry</selectedRelationships> | |
<source> | |
<groupId>fcff8588-6cae-364b-0000-000000000000</groupId> | |
<id>d870d851-d95d-3690-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<controllerServices> | |
<id>1af83330-0038-343b-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<bundle> | |
<artifact>nifi-dbcp-service-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.4.0-SNAPSHOT</version> | |
</bundle> | |
<comments></comments> | |
<descriptors> | |
<entry> | |
<key>Database Connection URL</key> | |
<value> | |
<name>Database Connection URL</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Database Driver Class Name</key> | |
<value> | |
<name>Database Driver Class Name</name> | |
</value> | |
</entry> | |
<entry> | |
<key>database-driver-locations</key> | |
<value> | |
<name>database-driver-locations</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Database User</key> | |
<value> | |
<name>Database User</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Password</key> | |
<value> | |
<name>Password</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value> | |
<name>Max Wait Time</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Total Connections</key> | |
<value> | |
<name>Max Total Connections</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Validation-query</key> | |
<value> | |
<name>Validation-query</name> | |
</value> | |
</entry> | |
</descriptors> | |
<name>DBCPConnectionPool</name> | |
<persistsState>false</persistsState> | |
<properties> | |
<entry> | |
<key>Database Connection URL</key> | |
<value>jdbc:mysql://192.168.99.100:3306/nifi_test</value> | |
</entry> | |
<entry> | |
<key>Database Driver Class Name</key> | |
<value>com.mysql.jdbc.Driver</value> | |
</entry> | |
<entry> | |
<key>database-driver-locations</key> | |
<value>/Users/koji/Downloads/mysql-connector-java-5.1.41/mysql-connector-java-5.1.41-bin.jar</value> | |
</entry> | |
<entry> | |
<key>Database User</key> | |
<value>root</value> | |
</entry> | |
<entry> | |
<key>Password</key> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
</entry> | |
<entry> | |
<key>Max Total Connections</key> | |
</entry> | |
<entry> | |
<key>Validation-query</key> | |
</entry> | |
</properties> | |
<state>ENABLED</state> | |
<type>org.apache.nifi.dbcp.DBCPConnectionPool</type> | |
</controllerServices> | |
<controllerServices> | |
<id>2fbf4c12-5ee6-34c5-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<bundle> | |
<artifact>nifi-record-serialization-services-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.4.0-SNAPSHOT</version> | |
</bundle> | |
<comments></comments> | |
<descriptors> | |
<entry> | |
<key>schema-access-strategy</key> | |
<value> | |
<name>schema-access-strategy</name> | |
</value> | |
</entry> | |
<entry> | |
<key>schema-registry</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.schemaregistry.services.SchemaRegistry</identifiesControllerService> | |
<name>schema-registry</name> | |
</value> | |
</entry> | |
<entry> | |
<key>schema-name</key> | |
<value> | |
<name>schema-name</name> | |
</value> | |
</entry> | |
<entry> | |
<key>schema-text</key> | |
<value> | |
<name>schema-text</name> | |
</value> | |
</entry> | |
</descriptors> | |
<name>AvroReader</name> | |
<persistsState>false</persistsState> | |
<properties> | |
<entry> | |
<key>schema-access-strategy</key> | |
</entry> | |
<entry> | |
<key>schema-registry</key> | |
</entry> | |
<entry> | |
<key>schema-name</key> | |
</entry> | |
<entry> | |
<key>schema-text</key> | |
</entry> | |
</properties> | |
<state>ENABLED</state> | |
<type>org.apache.nifi.avro.AvroReader</type> | |
</controllerServices> | |
<processors> | |
<id>d870d851-d95d-3690-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<position> | |
<x>936.9999962089732</x> | |
<y>661.9999942997559</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.4.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>put-db-record-record-reader</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.serialization.RecordReaderFactory</identifiesControllerService> | |
<name>put-db-record-record-reader</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-statement-type</key> | |
<value> | |
<name>put-db-record-statement-type</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-dcbp-service</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService> | |
<name>put-db-record-dcbp-service</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-catalog-name</key> | |
<value> | |
<name>put-db-record-catalog-name</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-schema-name</key> | |
<value> | |
<name>put-db-record-schema-name</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-table-name</key> | |
<value> | |
<name>put-db-record-table-name</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-translate-field-names</key> | |
<value> | |
<name>put-db-record-translate-field-names</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-unmatched-field-behavior</key> | |
<value> | |
<name>put-db-record-unmatched-field-behavior</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-unmatched-column-behavior</key> | |
<value> | |
<name>put-db-record-unmatched-column-behavior</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-update-keys</key> | |
<value> | |
<name>put-db-record-update-keys</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-field-containing-sql</key> | |
<value> | |
<name>put-db-record-field-containing-sql</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-quoted-identifiers</key> | |
<value> | |
<name>put-db-record-quoted-identifiers</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-quoted-table-identifiers</key> | |
<value> | |
<name>put-db-record-quoted-table-identifiers</name> | |
</value> | |
</entry> | |
<entry> | |
<key>put-db-record-query-timeout</key> | |
<value> | |
<name>put-db-record-query-timeout</name> | |
</value> | |
</entry> | |
<entry> | |
<key>rollback-on-failure</key> | |
<value> | |
<name>rollback-on-failure</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>put-db-record-record-reader</key> | |
<value>2fbf4c12-5ee6-34c5-0000-000000000000</value> | |
</entry> | |
<entry> | |
<key>put-db-record-statement-type</key> | |
<value>INSERT</value> | |
</entry> | |
<entry> | |
<key>put-db-record-dcbp-service</key> | |
<value>1af83330-0038-343b-0000-000000000000</value> | |
</entry> | |
<entry> | |
<key>put-db-record-catalog-name</key> | |
</entry> | |
<entry> | |
<key>put-db-record-schema-name</key> | |
</entry> | |
<entry> | |
<key>put-db-record-table-name</key> | |
<value>decimal_2</value> | |
</entry> | |
<entry> | |
<key>put-db-record-translate-field-names</key> | |
<value>true</value> | |
</entry> | |
<entry> | |
<key>put-db-record-unmatched-field-behavior</key> | |
<value>Ignore Unmatched Fields</value> | |
</entry> | |
<entry> | |
<key>put-db-record-unmatched-column-behavior</key> | |
<value>Fail on Unmatched Columns</value> | |
</entry> | |
<entry> | |
<key>put-db-record-update-keys</key> | |
</entry> | |
<entry> | |
<key>put-db-record-field-containing-sql</key> | |
</entry> | |
<entry> | |
<key>put-db-record-quoted-identifiers</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>put-db-record-quoted-table-identifiers</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>put-db-record-query-timeout</key> | |
<value>0 seconds</value> | |
</entry> | |
<entry> | |
<key>rollback-on-failure</key> | |
<value>false</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>PutDatabaseRecord</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>retry</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.standard.PutDatabaseRecord</type> | |
</processors> | |
<processors> | |
<id>137b0f09-d80a-3c68-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<position> | |
<x>343.9536705253795</x> | |
<y>806.0963077763184</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.4.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Log Level</key> | |
<value> | |
<name>Log Level</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Log Payload</key> | |
<value> | |
<name>Log Payload</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Attributes to Log</key> | |
<value> | |
<name>Attributes to Log</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Attributes to Ignore</key> | |
<value> | |
<name>Attributes to Ignore</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Log prefix</key> | |
<value> | |
<name>Log prefix</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Log Level</key> | |
<value>info</value> | |
</entry> | |
<entry> | |
<key>Log Payload</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>Attributes to Log</key> | |
</entry> | |
<entry> | |
<key>Attributes to Ignore</key> | |
</entry> | |
<entry> | |
<key>Log prefix</key> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>LogAttribute</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.standard.LogAttribute</type> | |
</processors> | |
<processors> | |
<id>60bec32b-990e-3aaf-0000-000000000000</id> | |
<parentGroupId>fcff8588-6cae-364b-0000-000000000000</parentGroupId> | |
<position> | |
<x>570.9999962089732</x> | |
<y>281.99999429975594</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.4.0-SNAPSHOT</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Database Connection Pooling Service</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService> | |
<name>Database Connection Pooling Service</name> | |
</value> | |
</entry> | |
<entry> | |
<key>SQL select query</key> | |
<value> | |
<name>SQL select query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value> | |
<name>Max Wait Time</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-normalize</key> | |
<value> | |
<name>dbf-normalize</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-user-logical-types</key> | |
<value> | |
<name>dbf-user-logical-types</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-default-precision</key> | |
<value> | |
<name>dbf-default-precision</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-default-scale</key> | |
<value> | |
<name>dbf-default-scale</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Database Connection Pooling Service</key> | |
<value>1af83330-0038-343b-0000-000000000000</value> | |
</entry> | |
<entry> | |
<key>SQL select query</key> | |
<value>select * from decimal_1</value> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value>0 seconds</value> | |
</entry> | |
<entry> | |
<key>dbf-normalize</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>dbf-user-logical-types</key> | |
<value>true</value> | |
</entry> | |
<entry> | |
<key>dbf-default-precision</key> | |
<value>10</value> | |
</entry> | |
<entry> | |
<key>dbf-default-scale</key> | |
<value>0</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>1d</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<name>ExecuteSQL</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<style></style> | |
<type>org.apache.nifi.processors.standard.ExecuteSQL</type> | |
</processors> | |
</contents> | |
<name>MySQL Decimal</name> | |
</processGroups> | |
</snippet> | |
<timestamp>06/13/2017 11:39:02 JST</timestamp> | |
</template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment