mshakhomirov

with mock as (
    select
-- a piece of real data:
'{"fields":{"customfield_10216":{"version":1,"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"Questions answered."},{"type":"hardBreak"},{"type":"text","text":"For GA4 customer have to use a difference granularity on free form report to match data with platform."},{"type":"hardBreak"},{"type":"text","text":"For Instagram as customer is using a lifetime report type they need to use the created_at_datetime field to find specific date range data."}]}]}}}'
    as _airbyte_data
)

, json_extract as (
select

mshakhomirov

with d as (



SELECT
  user_email,
  query,
  SUM(total_bytes_billed) AS bytes_billed
FROM

mshakhomirov

# Base image
#  tar -czvf awsgcpdocker4.tar.gz Dockerfile
FROM gcr.io/educative-exec-env/educative-ubuntu-microvm:latest

# Installing prerequisite packages
RUN DEBIAN_FRONTEND="noninteractive" apt-get update && apt-get install -y tzdata keyboard-configuration
RUN apt-get -y install curl unzip groff less zip git apt-transport-https ca-certificates gnupg
# AWS CLI installation commands
RUN	curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
RUN	unzip awscliv2.zip && ./aws/install

mshakhomirov

# .github/workflows/deploy_live.yaml
name: DEPLOY LIVE

on:
  workflow_dispatch:

jobs:
 compile:
   runs-on: ubuntu-latest
   steps:

mshakhomirov

# .github/workflows/deploy_staging.yaml
name: STAGING AND TESTS

on:
  #when there is a push to the master
  push:
    branches: [ master ]
  #when there is a pull to the master
  pull_request:
    branches: [ master ]

mshakhomirov

  AlarmNotificationTopic:
    Type: AWS::SNS::Topic
    Properties:
      Subscription:
      - Endpoint:
          Ref: Email
        Protocol: email
  PipelineManagerLambdaLogGroup:
    DeletionPolicy: Delete
    Type: AWS::Logs::LogGroup

mshakhomirov

# Deploy packaged Lambda using AWS CLI:
aws \
lambda create-function \
--function-name mysql-lambda \
--zip-file fileb://stack.zip \
--handler <path-to-your-lambda-handler>/app.lambda_handler \
--runtime python3.8 \
--role arn:aws:iam::<your-aws-account-id>:role/my-lambda-role

mshakhomirov

CREATE TABLE IF NOT EXISTS mydatabase.user_transactions (
  dt date,
  user_id int,
  total_cost_usd float,
  registration_date string
) 
PARTITIONED BY (dt)
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/' 
TBLPROPERTIES (
  'table_type'='ICEBERG',

mshakhomirov

CREATE EXTERNAL TABLE myexternal-table
WITH CONNECTION `myproject.us.myconnection`
OPTIONS (
        format = 'ICEBERG',
        uris = ["gs://mybucket/mydata/mytable/metadata/iceberg.metadata.json"]
)

mshakhomirov

CREATE TABLE IF NOT EXISTS mydatabase.user_transactions (
  dt date,
  user_id int,
  total_cost_usd float,
  registration_date string
) 
PARTITIONED BY (dt)
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/' 
TBLPROPERTIES (
  'table_type'='ICEBERG',