Skip to content

Instantly share code, notes, and snippets.

with d as (



SELECT
  user_email,
  query,
  SUM(total_bytes_billed) AS bytes_billed
FROM
# Base image
# tar -czvf awsgcpdocker4.tar.gz Dockerfile
FROM gcr.io/educative-exec-env/educative-ubuntu-microvm:latest
# Installing prerequisite packages
RUN DEBIAN_FRONTEND="noninteractive" apt-get update && apt-get install -y tzdata keyboard-configuration
RUN apt-get -y install curl unzip groff less zip git apt-transport-https ca-certificates gnupg
# AWS CLI installation commands
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
RUN unzip awscliv2.zip && ./aws/install
# .github/workflows/deploy_live.yaml
name: DEPLOY LIVE
on:
workflow_dispatch:
jobs:
compile:
runs-on: ubuntu-latest
steps:
# .github/workflows/deploy_staging.yaml
name: STAGING AND TESTS
on:
#when there is a push to the master
push:
branches: [ master ]
#when there is a pull to the master
pull_request:
branches: [ master ]
AlarmNotificationTopic:
Type: AWS::SNS::Topic
Properties:
Subscription:
- Endpoint:
Ref: Email
Protocol: email
PipelineManagerLambdaLogGroup:
DeletionPolicy: Delete
Type: AWS::Logs::LogGroup
# Deploy packaged Lambda using AWS CLI:
aws \
lambda create-function \
--function-name mysql-lambda \
--zip-file fileb://stack.zip \
--handler <path-to-your-lambda-handler>/app.lambda_handler \
--runtime python3.8 \
--role arn:aws:iam::<your-aws-account-id>:role/my-lambda-role
CREATE TABLE IF NOT EXISTS mydatabase.user_transactions (
dt date,
user_id int,
total_cost_usd float,
registration_date string
)
PARTITIONED BY (dt)
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/'
TBLPROPERTIES (
'table_type'='ICEBERG',
CREATE EXTERNAL TABLE myexternal-table
WITH CONNECTION `myproject.us.myconnection`
OPTIONS (
format = 'ICEBERG',
uris = ["gs://mybucket/mydata/mytable/metadata/iceberg.metadata.json"]
)
CREATE TABLE IF NOT EXISTS mydatabase.user_transactions (
dt date,
user_id int,
total_cost_usd float,
registration_date string
)
PARTITIONED BY (dt)
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/'
TBLPROPERTIES (
'table_type'='ICEBERG',
# simple_stack.yaml
# AWS Cloudformation example for data pipeline
AWSTemplateFormatVersion: '2010-09-09'
Description: An example template for a Step Functions state machine.
Resources:
LambdaExecutionRole:
Type: "AWS::IAM::Role"
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"