with d as (
SELECT
user_email,
query,
SUM(total_bytes_billed) AS bytes_billed
FROM
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Base image | |
# tar -czvf awsgcpdocker4.tar.gz Dockerfile | |
FROM gcr.io/educative-exec-env/educative-ubuntu-microvm:latest | |
# Installing prerequisite packages | |
RUN DEBIAN_FRONTEND="noninteractive" apt-get update && apt-get install -y tzdata keyboard-configuration | |
RUN apt-get -y install curl unzip groff less zip git apt-transport-https ca-certificates gnupg | |
# AWS CLI installation commands | |
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
RUN unzip awscliv2.zip && ./aws/install |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# .github/workflows/deploy_live.yaml | |
name: DEPLOY LIVE | |
on: | |
workflow_dispatch: | |
jobs: | |
compile: | |
runs-on: ubuntu-latest | |
steps: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# .github/workflows/deploy_staging.yaml | |
name: STAGING AND TESTS | |
on: | |
#when there is a push to the master | |
push: | |
branches: [ master ] | |
#when there is a pull to the master | |
pull_request: | |
branches: [ master ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
AlarmNotificationTopic: | |
Type: AWS::SNS::Topic | |
Properties: | |
Subscription: | |
- Endpoint: | |
Ref: Email | |
Protocol: email | |
PipelineManagerLambdaLogGroup: | |
DeletionPolicy: Delete | |
Type: AWS::Logs::LogGroup |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Deploy packaged Lambda using AWS CLI: | |
aws \ | |
lambda create-function \ | |
--function-name mysql-lambda \ | |
--zip-file fileb://stack.zip \ | |
--handler <path-to-your-lambda-handler>/app.lambda_handler \ | |
--runtime python3.8 \ | |
--role arn:aws:iam::<your-aws-account-id>:role/my-lambda-role |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE TABLE IF NOT EXISTS mydatabase.user_transactions ( | |
dt date, | |
user_id int, | |
total_cost_usd float, | |
registration_date string | |
) | |
PARTITIONED BY (dt) | |
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/' | |
TBLPROPERTIES ( | |
'table_type'='ICEBERG', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE EXTERNAL TABLE myexternal-table | |
WITH CONNECTION `myproject.us.myconnection` | |
OPTIONS ( | |
format = 'ICEBERG', | |
uris = ["gs://mybucket/mydata/mytable/metadata/iceberg.metadata.json"] | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE TABLE IF NOT EXISTS mydatabase.user_transactions ( | |
dt date, | |
user_id int, | |
total_cost_usd float, | |
registration_date string | |
) | |
PARTITIONED BY (dt) | |
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/' | |
TBLPROPERTIES ( | |
'table_type'='ICEBERG', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# simple_stack.yaml | |
# AWS Cloudformation example for data pipeline | |
AWSTemplateFormatVersion: '2010-09-09' | |
Description: An example template for a Step Functions state machine. | |
Resources: | |
LambdaExecutionRole: | |
Type: "AWS::IAM::Role" | |
Properties: | |
AssumeRolePolicyDocument: | |
Version: "2012-10-17" |
NewerOlder