Skip to content

Instantly share code, notes, and snippets.

View aescay's full-sized avatar
👨‍💻
Exploring Exploring Exploring

Andrew Escay aescay

👨‍💻
Exploring Exploring Exploring
View GitHub Profile
@aescay
aescay / should_full_refresh.sql
Created June 24, 2024 15:52
dbt Labs' should_full_refresh macro override
{# this is overriding the default macro to support our specialized incremental runs #}
{# this first macro is just a mirror of the default should_full_refresh logic
which can be found in the dbt-snowflake adaptor repo #}
{% macro is_full_refresh_flagged() %}
{% set raw_config_full_refresh = config.get('full_refresh') %}
{% if raw_config_full_refresh is none %}
{% set raw_config_full_refresh = flags.FULL_REFRESH %}
{% endif %}
@aescay
aescay / incremental_filters.sql
Created June 24, 2024 15:51
dbt Labs' Enhanced Incremental Models Standard Filters
{% macro enhanced_incremental_filters(ref_timestamp, this_timestamp, day_overwrite=false) %}
{% if target.name in var('dev_targets') %}
and {{ ref_timestamp }} >= dateadd(d, -{{ var('testing_days_of_data') }}, current_date)
{% elif is_incremental() and not is_full_refresh_flagged() %}
{% if day_overwrite %}
and {{ ref_timestamp }} > (select dateadd(day, -1, max({{ this_timestamp }})) from {{ this }})
{% else %}
and {{ ref_timestamp }} >= (select max({{ this_timestamp }}) from {{ this }})
{% endif %}
@aescay
aescay / should_full_refresh_demo.sql
Created June 24, 2024 15:49
dbt on dbt Japan: should full refresh override demo
{# this is overriding the default macro to support our specialized incremental runs #}
{# this first macro is just a mirror of the default should_full_refresh logic
which can be found in the dbt-snowflake adaptor repo #}
{% macro is_full_refresh_flagged() %}
{% set raw_config_full_refresh = config.get('full_refresh') %}
{% if raw_config_full_refresh is none %}
{% set raw_config_full_refresh = flags.FULL_REFRESH %}
{% endif %}
@aescay
aescay / fct_daily_active_users_final.sql
Created June 24, 2024 15:48
dbt on dbt Japan: final incremental model demo code
{{
config(
materialized='incremental',
unique_key='date_day',
cold_storage=true
)
}}
with
#!/usr/bin/env bash
DBT_ENV=~/.virtualenvs/dbt
DBT_BETA_ENV=~/.virtualenvs/dbt-beta
process_environment() {
env=$1
release=$2
if [[ -d "$env" ]]; then