mshakhomirov

CREATE EXTERNAL TABLE myexternal-table
WITH CONNECTION `myproject.us.myconnection`
OPTIONS (
        format = 'ICEBERG',
        uris = ["gs://mybucket/mydata/mytable/metadata/iceberg.metadata.json"]
)

mshakhomirov

CREATE TABLE IF NOT EXISTS mydatabase.user_transactions (
  dt date,
  user_id int,
  total_cost_usd float,
  registration_date string
) 
PARTITIONED BY (dt)
LOCATION 's3://datalake.staging.aws/data/myschema/optimized-data-iceberg-parquet/' 
TBLPROPERTIES (
  'table_type'='ICEBERG',

mshakhomirov

# simple_stack.yaml
# AWS Cloudformation example for data pipeline
AWSTemplateFormatVersion: '2010-09-09'
Description: An example template for a Step Functions state machine.
Resources:
  LambdaExecutionRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"

mshakhomirov

aws \
cloudformation deploy \
--template-file simple_stack.yaml \
--stack-name SimpleDataPipeline \
--capabilities CAPABILITY_IAM

mshakhomirov

AWSTemplateFormatVersion: 2010-09-09
Description: >-
  AWS CloudFormation Sample Template: Sample template showing how to
  create an Amazon RDS Database Instance with provisioned IOPs.**WARNING** This
  template creates an Amazon Relational Database Service database instance. You
  will be billed for the AWS resources used if you create a stack from this
  template.
Parameters:
  DBUser:
    Default: root

mshakhomirov

AWSTemplateFormatVersion: 2010-09-09
Description: >-
  AWS CloudFormation Sample Template: Sample template showing how to
  create an Amazon RDS Database Instance with provisioned IOPs.**WARNING** This
  template creates an Amazon Relational Database Service database instance. You
  will be billed for the AWS resources used if you create a stack from this
  template.
Parameters:
  DBUser:
    Default: root

mshakhomirov

aws \
cloudformation deploy \
--template-file AWS_cfn_mysql_small.yaml \
--stack-name MyMySQLDB \
--capabilities CAPABILITY_IAM

mshakhomirov

aws --profile mds \
cloudformation deploy \
--template-file AWS_cfn_postgres_micro.yaml \
--stack-name MyPostgresDB \
--capabilities CAPABILITY_IAM

mshakhomirov

aws --profile mds \
cloudformation deploy \
--template-file AWS_cfn_postgres_micro.yaml \
--stack-name MyMySQLDB \
--capabilities CAPABILITY_IAM

mshakhomirov

USE gcs_hive.default;
CREATE TABLE gcs_hive.fb.events_raw
WITH (
     external_location = 'gs://firebase-events-archive-avro',
     format = 'AVRO' -- or 'PARQUET'
);

CREATE SCHEMA gcs_hive.fb WITH (location = 'gs://firebase-events-archive-avro');