JeffreyMFarley

# Generated by ChatGPT; Modified to work by me

import json
from datetime import datetime, timedelta
from airflow.providers.amazon.aws.sensors.sqs import SqsSensor
from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
from airflow.operators.empty import EmptyOperator
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago
from csm_snowflake_elt.constants import MANAGER, SNOWFLAKE_CONNECTION

JeffreyMFarley

# https://chat.openai.com/c/9d7c6f7d-2dbf-4d29-a2d2-364c6750d476

import boto3
import csv
import pytz
from collections import Counter, OrderedDict
from datetime import datetime
from io import StringIO

BUCKET_NAME = "foo"

JeffreyMFarley

import random

from snowflake.snowpark.functions import trunc

def model(dbt, session):
    source_df = dbt.ref("datorama_v2_temp")
    campaigns = source_df[['campaign_id', 'day', 'impressions']]

    campaigns = campaigns.withColumn(
        'projected_five_sec_views',

JeffreyMFarley

select 
  a.table_name, 
  b.column_name,
  c.constraint_name

from information_schema.tables a
inner join information_schema.columns b
        on a.table_name = b.table_name
inner join information_schema.key_column_usage c
        on b.column_name = c.column_name and a.table_name = c.table_name

JeffreyMFarley

from datetime import date, datetime
from decimal import Decimal

def json_serial(obj):
    """Helps json.dumps render datetime objects"""
    if isinstance(obj, date):
        return obj.isoformat()[:10]
    if isinstance(obj, datetime):
        return obj.isoformat()[:10]
    if isinstance(obj, Decimal):

JeffreyMFarley

from scout_apm.core.tracked_request import TrackedRequest

def execute(self, sql, *args, **kwargs):
  tracked_request = TrackedRequest.instance()
  span = tracked_request.start_span(operation='SQL/Query')
  span.tag("db.statement", sql)
  self._inner.execute(sql, *args, **kwargs)
  tracked_request.stop_span()

JeffreyMFarley

How should we clean up our branches? We have a lot

git branch -r --merged shows the remote branches that have been merged to main

git branch -r --no-merged shows the remote branches that have not been merged to main. These are either abandoned or squash-merged

Then git push origin --delete

JeffreyMFarley

# Modified from https://cscheid.net/2017/12/11/minimal-tracing-decorator-python-3.html

import inspect

trace_indent = 0
def tracing(f):
    sig = inspect.signature(f)
    def do_it(*args, **kwargs):
        global trace_indent
        ws = ' ' * (trace_indent * 2)

JeffreyMFarley

Prerequistites

yq

.zshrc

export YQFMT_DEFENV="{\"defaults\": .defaults | sort_by(.key), \"mapping\": .mapping | sort_keys(.), \"services\": .services | sort_keys(.)}"
export YQFMT_ENV="{\"envs\": .envs | sort_by(.key), \"mapping\": .mapping | sort_keys(.), \"services\": .services | sort_keys(.)}"

JeffreyMFarley

Run a privately-built image in a GitHub action

1. https://github.com/marketplace/actions/amazon-ecr-login-action-for-github-actions
   1. https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services
2. https://agileek.github.io/software/aws/2020/10/28/using-an-ecr-image-in-github-actions/
3. https://gist.github.com/eucleciojosias/02e37509e0771df708fdb9d5baeba464

Create the Role in AWS

1. Create a Identity Provider