Last active
December 6, 2021 00:35
-
-
Save rtempleton/55b3103ce4be6f2b697e78e99f82acf8 to your computer and use it in GitHub Desktop.
This is an importable Nifi template that can be used to monitor Snowflake resources and send reports via Slack. https://medium.com/@ryan_templeton/snowflake-resource-monitor-reports-delivered-to-slack-using-apache-nifi-8dfd4fc4d579
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0" encoding="UTF-8" standalone="yes"?> | |
<template encoding-version="1.2"> | |
<description>Workflow to monitor Snowflake credit consumption and report to Slack. Full article can be found here https://snowflakecomputing.atlassian.net/wiki/spaces/RT/pages/771200336/Creating+a+Snowflake+usage+report+using+Apache+Nifi | |
Requires existing tables and apps defined in your Snowflake account and your Slack account</description> | |
<groupId>be279da7-0167-1000-6a67-4d0cb36decc6</groupId> | |
<name>SnowflakeCreditMonitoring</name> | |
<snippet> | |
<connections> | |
<id>1d50d3ac-3446-3e09-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>1</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>7ce571dd-6658-3047-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>matched</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>8df955bc-1bdf-3ab8-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>218dec3c-925c-325f-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>6b154671-6aa4-3ede-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>0d88a793-090c-35e5-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>22f8f5ae-d728-317d-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>4dd4c26d-8b16-35e5-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>f36db6ed-77d4-3b96-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>5ffbfa2f-fa57-3fde-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>47b3c036-b571-3ec7-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>6b154671-6aa4-3ede-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>ace01620-fed7-36ab-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>0d88a793-090c-35e5-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>3790f5c6-b878-39e3-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>c9f65e53-590e-3cce-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>abbd997b-0d38-3512-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>4dd4c26d-8b16-35e5-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>ee95c725-d8cb-33b6-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>6b154671-6aa4-3ede-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>matched</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>abbd997b-0d38-3512-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<connections> | |
<id>f26b34df-f273-38f3-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> | |
<backPressureObjectThreshold>10000</backPressureObjectThreshold> | |
<destination> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>8df955bc-1bdf-3ab8-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</destination> | |
<flowFileExpiration>0 sec</flowFileExpiration> | |
<labelIndex>1</labelIndex> | |
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> | |
<loadBalancePartitionAttribute></loadBalancePartitionAttribute> | |
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus> | |
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> | |
<name></name> | |
<selectedRelationships>success</selectedRelationships> | |
<source> | |
<groupId>9352e60a-8d45-3f92-0000-000000000000</groupId> | |
<id>47b3c036-b571-3ec7-0000-000000000000</id> | |
<type>PROCESSOR</type> | |
</source> | |
<zIndex>0</zIndex> | |
</connections> | |
<controllerServices> | |
<id>cd121fa3-547b-37b3-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<bundle> | |
<artifact>nifi-dbcp-service-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<comments></comments> | |
<descriptors> | |
<entry> | |
<key>Database Connection URL</key> | |
<value> | |
<name>Database Connection URL</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Database Driver Class Name</key> | |
<value> | |
<name>Database Driver Class Name</name> | |
</value> | |
</entry> | |
<entry> | |
<key>database-driver-locations</key> | |
<value> | |
<name>database-driver-locations</name> | |
</value> | |
</entry> | |
<entry> | |
<key>kerberos-credentials-service</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.kerberos.KerberosCredentialsService</identifiesControllerService> | |
<name>kerberos-credentials-service</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Database User</key> | |
<value> | |
<name>Database User</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Password</key> | |
<value> | |
<name>Password</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value> | |
<name>Max Wait Time</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Total Connections</key> | |
<value> | |
<name>Max Total Connections</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Validation-query</key> | |
<value> | |
<name>Validation-query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbcp-min-idle-conns</key> | |
<value> | |
<name>dbcp-min-idle-conns</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbcp-max-idle-conns</key> | |
<value> | |
<name>dbcp-max-idle-conns</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbcp-max-conn-lifetime</key> | |
<value> | |
<name>dbcp-max-conn-lifetime</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbcp-time-between-eviction-runs</key> | |
<value> | |
<name>dbcp-time-between-eviction-runs</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbcp-min-evictable-idle-time</key> | |
<value> | |
<name>dbcp-min-evictable-idle-time</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbcp-soft-min-evictable-idle-time</key> | |
<value> | |
<name>dbcp-soft-min-evictable-idle-time</name> | |
</value> | |
</entry> | |
</descriptors> | |
<name>GlobalSnowflakeConPool</name> | |
<persistsState>false</persistsState> | |
<properties> | |
<entry> | |
<key>Database Connection URL</key> | |
<value>jdbc:snowflake://MyAccount.snowflakecomputing.com/?db=MY_TEST_DB&schema=BB&warehouse=ELT_WH</value> | |
</entry> | |
<entry> | |
<key>Database Driver Class Name</key> | |
<value>net.snowflake.client.jdbc.SnowflakeDriver</value> | |
</entry> | |
<entry> | |
<key>database-driver-locations</key> | |
<value>/Users/rtempleton/Devtools/JDBC/Snowflake/snowflake-jdbc-3.9.2.jar</value> | |
</entry> | |
<entry> | |
<key>kerberos-credentials-service</key> | |
</entry> | |
<entry> | |
<key>Database User</key> | |
<value>snowman</value> | |
</entry> | |
<entry> | |
<key>Password</key> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value>500 millis</value> | |
</entry> | |
<entry> | |
<key>Max Total Connections</key> | |
<value>16</value> | |
</entry> | |
<entry> | |
<key>Validation-query</key> | |
<value>describe table "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."NATION"</value> | |
</entry> | |
<entry> | |
<key>dbcp-min-idle-conns</key> | |
<value>0</value> | |
</entry> | |
<entry> | |
<key>dbcp-max-idle-conns</key> | |
<value>8</value> | |
</entry> | |
<entry> | |
<key>dbcp-max-conn-lifetime</key> | |
<value>24 hrs</value> | |
</entry> | |
<entry> | |
<key>dbcp-time-between-eviction-runs</key> | |
<value>10 mins</value> | |
</entry> | |
<entry> | |
<key>dbcp-min-evictable-idle-time</key> | |
<value>30 mins</value> | |
</entry> | |
<entry> | |
<key>dbcp-soft-min-evictable-idle-time</key> | |
<value>-1</value> | |
</entry> | |
</properties> | |
<state>ENABLED</state> | |
<type>org.apache.nifi.dbcp.DBCPConnectionPool</type> | |
</controllerServices> | |
<processors> | |
<id>0d88a793-090c-35e5-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>587.8049980119222</x> | |
<y>210.01627612937057</y> | |
</position> | |
<bundle> | |
<artifact>nifi-update-attribute-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
<value> | |
<name>Delete Attributes Expression</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value> | |
<name>Store State</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
<value> | |
<name>Stateful Variables Initial Value</name> | |
</value> | |
</entry> | |
<entry> | |
<key>canonical-value-lookup-cache-size</key> | |
<value> | |
<name>canonical-value-lookup-cache-size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>report.type</key> | |
<value> | |
<name>report.type</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value>Do not store state</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
</entry> | |
<entry> | |
<key>canonical-value-lookup-cache-size</key> | |
<value>100</value> | |
</entry> | |
<entry> | |
<key>report.type</key> | |
<value>:blue-snowflake: Usage Report</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>UpdateAttribute</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
</processors> | |
<processors> | |
<id>3790f5c6-b878-39e3-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.31390380859375</x> | |
<y>209.63177344075223</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Database Connection Pooling Service</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService> | |
<name>Database Connection Pooling Service</name> | |
</value> | |
</entry> | |
<entry> | |
<key>sql-pre-query</key> | |
<value> | |
<name>sql-pre-query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>SQL select query</key> | |
<value> | |
<name>SQL select query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>sql-post-query</key> | |
<value> | |
<name>sql-post-query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value> | |
<name>Max Wait Time</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-normalize</key> | |
<value> | |
<name>dbf-normalize</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-user-logical-types</key> | |
<value> | |
<name>dbf-user-logical-types</name> | |
</value> | |
</entry> | |
<entry> | |
<key>compression-format</key> | |
<value> | |
<name>compression-format</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-default-precision</key> | |
<value> | |
<name>dbf-default-precision</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-default-scale</key> | |
<value> | |
<name>dbf-default-scale</name> | |
</value> | |
</entry> | |
<entry> | |
<key>esql-max-rows</key> | |
<value> | |
<name>esql-max-rows</name> | |
</value> | |
</entry> | |
<entry> | |
<key>esql-output-batch-size</key> | |
<value> | |
<name>esql-output-batch-size</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Database Connection Pooling Service</key> | |
<value>cd121fa3-547b-37b3-0000-000000000000</value> | |
</entry> | |
<entry> | |
<key>sql-pre-query</key> | |
</entry> | |
<entry> | |
<key>SQL select query</key> | |
<value>select a.*, iff(b.credits_billed is null, 0, b.credits_billed)::string as credits_billed from | |
"MY_TEST_DB"."PUBLIC"."THRESHOLDS" a left join | |
(select service_type, sum(credits_billed::double) as credits_billed from "SNOWFLAKE"."ACCOUNT_USAGE"."METERING_DAILY_HISTORY" where usage_date=current_date() group by 1) b | |
on a.service_type = b.service_type;</value> | |
</entry> | |
<entry> | |
<key>sql-post-query</key> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value>0 seconds</value> | |
</entry> | |
<entry> | |
<key>dbf-normalize</key> | |
<value>true</value> | |
</entry> | |
<entry> | |
<key>dbf-user-logical-types</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>compression-format</key> | |
<value>NONE</value> | |
</entry> | |
<entry> | |
<key>dbf-default-precision</key> | |
<value>2</value> | |
</entry> | |
<entry> | |
<key>dbf-default-scale</key> | |
<value>0</value> | |
</entry> | |
<entry> | |
<key>esql-max-rows</key> | |
<value>0</value> | |
</entry> | |
<entry> | |
<key>esql-output-batch-size</key> | |
<value>0</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>1 min</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>Hourly Report</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.ExecuteSQL</type> | |
</processors> | |
<processors> | |
<id>47b3c036-b571-3ec7-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>1177.731197881887</x> | |
<y>428.2860005821386</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>jolt-transform</key> | |
<value> | |
<name>jolt-transform</name> | |
</value> | |
</entry> | |
<entry> | |
<key>jolt-custom-class</key> | |
<value> | |
<name>jolt-custom-class</name> | |
</value> | |
</entry> | |
<entry> | |
<key>jolt-custom-modules</key> | |
<value> | |
<name>jolt-custom-modules</name> | |
</value> | |
</entry> | |
<entry> | |
<key>jolt-spec</key> | |
<value> | |
<name>jolt-spec</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Transform Cache Size</key> | |
<value> | |
<name>Transform Cache Size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>pretty_print</key> | |
<value> | |
<name>pretty_print</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>jolt-transform</key> | |
<value>jolt-transform-chain</value> | |
</entry> | |
<entry> | |
<key>jolt-custom-class</key> | |
</entry> | |
<entry> | |
<key>jolt-custom-modules</key> | |
</entry> | |
<entry> | |
<key>jolt-spec</key> | |
<value>[ | |
{ | |
"operation": "shift", | |
"spec": { | |
"*": { | |
"SERVICE_TYPE": "blocks[1].fields[].text", | |
"CREDITS_BILLED": "blocks[1].fields[].text" | |
} | |
} | |
}, | |
{ | |
"operation": "default", | |
"spec": { | |
"blocks[]": { | |
"0": { | |
"type": "section", | |
"fields[]": { | |
"0": { | |
"type": "mrkdwn", | |
"text": "*SERVICE TYPE*" | |
}, | |
"1": { | |
"type": "mrkdwn", | |
"text": "*CREDITS USED*" | |
} | |
} | |
}, | |
"1": { | |
"type": "section", | |
"fields[]": { | |
"*": { | |
"type": "plain_text" | |
} | |
} | |
} | |
} | |
} | |
} | |
] | |
</value> | |
</entry> | |
<entry> | |
<key>Transform Cache Size</key> | |
<value>1</value> | |
</entry> | |
<entry> | |
<key>pretty_print</key> | |
<value>false</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>JoltTransformJSON</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.JoltTransformJSON</type> | |
</processors> | |
<processors> | |
<id>4dd4c26d-8b16-35e5-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>586.7155204728597</x> | |
<y>23.576090582495567</y> | |
</position> | |
<bundle> | |
<artifact>nifi-update-attribute-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
<value> | |
<name>Delete Attributes Expression</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value> | |
<name>Store State</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
<value> | |
<name>Stateful Variables Initial Value</name> | |
</value> | |
</entry> | |
<entry> | |
<key>canonical-value-lookup-cache-size</key> | |
<value> | |
<name>canonical-value-lookup-cache-size</name> | |
</value> | |
</entry> | |
<entry> | |
<key>report.type</key> | |
<value> | |
<name>report.type</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Delete Attributes Expression</key> | |
</entry> | |
<entry> | |
<key>Store State</key> | |
<value>Do not store state</value> | |
</entry> | |
<entry> | |
<key>Stateful Variables Initial Value</key> | |
</entry> | |
<entry> | |
<key>canonical-value-lookup-cache-size</key> | |
<value>100</value> | |
</entry> | |
<entry> | |
<key>report.type</key> | |
<value>:alert: Usage Alert</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>UpdateAttribute</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type> | |
</processors> | |
<processors> | |
<id>6b154671-6aa4-3ede-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>1167.4050449599408</x> | |
<y>224.5679264760539</y> | |
</position> | |
<bundle> | |
<artifact>nifi-avro-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>JSON container options</key> | |
<value> | |
<name>JSON container options</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Wrap Single Record</key> | |
<value> | |
<name>Wrap Single Record</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Avro schema</key> | |
<value> | |
<name>Avro schema</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>JSON container options</key> | |
<value>array</value> | |
</entry> | |
<entry> | |
<key>Wrap Single Record</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>Avro schema</key> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>ConvertAvroToJSON</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.avro.ConvertAvroToJSON</type> | |
</processors> | |
<processors> | |
<id>7ce571dd-6658-3047-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>584.539009948764</x> | |
<y>631.4595129452019</y> | |
</position> | |
<bundle> | |
<artifact>nifi-slack-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>webhook-url</key> | |
<value> | |
<name>webhook-url</name> | |
</value> | |
</entry> | |
<entry> | |
<key>webhook-text</key> | |
<value> | |
<name>webhook-text</name> | |
</value> | |
</entry> | |
<entry> | |
<key>channel</key> | |
<value> | |
<name>channel</name> | |
</value> | |
</entry> | |
<entry> | |
<key>username</key> | |
<value> | |
<name>username</name> | |
</value> | |
</entry> | |
<entry> | |
<key>icon-url</key> | |
<value> | |
<name>icon-url</name> | |
</value> | |
</entry> | |
<entry> | |
<key>icon-emoji</key> | |
<value> | |
<name>icon-emoji</name> | |
</value> | |
</entry> | |
<entry> | |
<key>bar</key> | |
<value> | |
<name>bar</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>webhook-url</key> | |
</entry> | |
<entry> | |
<key>webhook-text</key> | |
<value>${report.type}</value> | |
</entry> | |
<entry> | |
<key>channel</key> | |
</entry> | |
<entry> | |
<key>username</key> | |
</entry> | |
<entry> | |
<key>icon-url</key> | |
</entry> | |
<entry> | |
<key>icon-emoji</key> | |
</entry> | |
<entry> | |
<key>bar</key> | |
<value>${message.body}</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>10 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>PutSlack</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.slack.PutSlack</type> | |
</processors> | |
<processors> | |
<id>8df955bc-1bdf-3ab8-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>585.4169353107918</x> | |
<y>405.14730205524404</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Destination</key> | |
<value> | |
<name>Destination</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Return Type</key> | |
<value> | |
<name>Return Type</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Path Not Found Behavior</key> | |
<value> | |
<name>Path Not Found Behavior</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Null Value Representation</key> | |
<value> | |
<name>Null Value Representation</name> | |
</value> | |
</entry> | |
<entry> | |
<key>message.body</key> | |
<value> | |
<name>message.body</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Destination</key> | |
<value>flowfile-attribute</value> | |
</entry> | |
<entry> | |
<key>Return Type</key> | |
<value>json</value> | |
</entry> | |
<entry> | |
<key>Path Not Found Behavior</key> | |
<value>ignore</value> | |
</entry> | |
<entry> | |
<key>Null Value Representation</key> | |
<value>empty string</value> | |
</entry> | |
<entry> | |
<key>message.body</key> | |
<value>$</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>EvaluateJsonPath</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>matched</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>unmatched</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.EvaluateJsonPath</type> | |
</processors> | |
<processors> | |
<id>abbd997b-0d38-3512-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>1164.9112803057249</x> | |
<y>0.0</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Routing Strategy</key> | |
<value> | |
<name>Routing Strategy</name> | |
</value> | |
</entry> | |
<entry> | |
<key>matched</key> | |
<value> | |
<name>matched</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Routing Strategy</key> | |
<value>Route to Property name</value> | |
</entry> | |
<entry> | |
<key>matched</key> | |
<value>${executesql.row.count:gt(0)}</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>0 sec</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>RouteOnAttribute</name> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>matched</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>unmatched</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.RouteOnAttribute</type> | |
</processors> | |
<processors> | |
<id>f36db6ed-77d4-3b96-0000-000000000000</id> | |
<parentGroupId>9352e60a-8d45-3f92-0000-000000000000</parentGroupId> | |
<position> | |
<x>0.0</x> | |
<y>16.991514651689727</y> | |
</position> | |
<bundle> | |
<artifact>nifi-standard-nar</artifact> | |
<group>org.apache.nifi</group> | |
<version>1.9.2</version> | |
</bundle> | |
<config> | |
<bulletinLevel>WARN</bulletinLevel> | |
<comments></comments> | |
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> | |
<descriptors> | |
<entry> | |
<key>Database Connection Pooling Service</key> | |
<value> | |
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService> | |
<name>Database Connection Pooling Service</name> | |
</value> | |
</entry> | |
<entry> | |
<key>sql-pre-query</key> | |
<value> | |
<name>sql-pre-query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>SQL select query</key> | |
<value> | |
<name>SQL select query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>sql-post-query</key> | |
<value> | |
<name>sql-post-query</name> | |
</value> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value> | |
<name>Max Wait Time</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-normalize</key> | |
<value> | |
<name>dbf-normalize</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-user-logical-types</key> | |
<value> | |
<name>dbf-user-logical-types</name> | |
</value> | |
</entry> | |
<entry> | |
<key>compression-format</key> | |
<value> | |
<name>compression-format</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-default-precision</key> | |
<value> | |
<name>dbf-default-precision</name> | |
</value> | |
</entry> | |
<entry> | |
<key>dbf-default-scale</key> | |
<value> | |
<name>dbf-default-scale</name> | |
</value> | |
</entry> | |
<entry> | |
<key>esql-max-rows</key> | |
<value> | |
<name>esql-max-rows</name> | |
</value> | |
</entry> | |
<entry> | |
<key>esql-output-batch-size</key> | |
<value> | |
<name>esql-output-batch-size</name> | |
</value> | |
</entry> | |
</descriptors> | |
<executionNode>ALL</executionNode> | |
<lossTolerant>false</lossTolerant> | |
<penaltyDuration>30 sec</penaltyDuration> | |
<properties> | |
<entry> | |
<key>Database Connection Pooling Service</key> | |
<value>cd121fa3-547b-37b3-0000-000000000000</value> | |
</entry> | |
<entry> | |
<key>sql-pre-query</key> | |
</entry> | |
<entry> | |
<key>SQL select query</key> | |
<value>select service_type, credits_billed from ( | |
select a.*, iff(b.credits_billed is null, 0, b.credits_billed)::string as credits_billed from | |
"MY_TEST_DB"."PUBLIC"."THRESHOLDS" a left join | |
(select service_type, sum(credits_billed::double) as credits_billed from "SNOWFLAKE"."ACCOUNT_USAGE"."METERING_DAILY_HISTORY" where usage_date=current_date() group by 1) b | |
on a.service_type = b.service_type | |
) where credits_billed > credits_threshold</value> | |
</entry> | |
<entry> | |
<key>sql-post-query</key> | |
</entry> | |
<entry> | |
<key>Max Wait Time</key> | |
<value>0 seconds</value> | |
</entry> | |
<entry> | |
<key>dbf-normalize</key> | |
<value>true</value> | |
</entry> | |
<entry> | |
<key>dbf-user-logical-types</key> | |
<value>false</value> | |
</entry> | |
<entry> | |
<key>compression-format</key> | |
<value>NONE</value> | |
</entry> | |
<entry> | |
<key>dbf-default-precision</key> | |
<value>2</value> | |
</entry> | |
<entry> | |
<key>dbf-default-scale</key> | |
<value>0</value> | |
</entry> | |
<entry> | |
<key>esql-max-rows</key> | |
<value>0</value> | |
</entry> | |
<entry> | |
<key>esql-output-batch-size</key> | |
<value>0</value> | |
</entry> | |
</properties> | |
<runDurationMillis>0</runDurationMillis> | |
<schedulingPeriod>1min</schedulingPeriod> | |
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> | |
<yieldDuration>1 sec</yieldDuration> | |
</config> | |
<executionNodeRestricted>false</executionNodeRestricted> | |
<name>15 minute threshold check</name> | |
<relationships> | |
<autoTerminate>true</autoTerminate> | |
<name>failure</name> | |
</relationships> | |
<relationships> | |
<autoTerminate>false</autoTerminate> | |
<name>success</name> | |
</relationships> | |
<state>STOPPED</state> | |
<style/> | |
<type>org.apache.nifi.processors.standard.ExecuteSQL</type> | |
</processors> | |
</snippet> | |
<timestamp>02/17/2020 10:56:13 PST</timestamp> | |
</template> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment