anna-anisienia

SELECT
  resourceId,
  resourceType,
  awsRegion,
  resourceCreationTime,
  tags,
  tags.tag,
  configuration.state.value
WHERE
  resourceType NOT IN ('AWS::EC2::SecurityGroup',

anna-anisienia

import requests
from pprint import pprint

base_url = "http://localhost:3000/api"
auth = {"Authorization": "Bearer YWRtaW46Ym9vdHN0cmFwOmFkbWluMTIz"}
workspaces = requests.get(f"{base_url}/entities/workspaces", headers=auth)
data = workspaces.json().get("data")
workspace_id = data[0].get("id")

dashboards = requests.get(f"{base_url}/entities/workspaces/{workspace_id}/analyticalDashboards",

anna-anisienia

"""
Dataset used: https://datasets.wri.org/dataset/globalpowerplantdatabase
"""
import os
import time
import pandas as pd
from datetime import datetime, timedelta
import awswrangler as wr
from boto3.dynamodb.conditions import Key

anna-anisienia

"""
Dataset used: https://datasets.wri.org/dataset/globalpowerplantdatabase
"""
import os
import time
import pandas as pd
from datetime import datetime, timedelta
import awswrangler as wr
from boto3.dynamodb.conditions import Key

anna-anisienia

import os
import boto3
import tempfile
from airflow import DAG
from airflow.decorators import task
from airflow.utils.dates import days_ago
from airflow.providers.amazon.aws.transfers.s3_to_redshift import S3ToRedshiftOperator
from airflow.utils.task_group import TaskGroup

S3_BUCKET = 'test_bucket_123'

anna-anisienia

aws ecr create-repository --repository-name lambda-docker-etl --image-scanning-configuration scanOnPush=true

docker tag lambda-docker-etl:latest 123456789.dkr.ecr.eu-central-1.amazonaws.com/lambda-docker-etl:latest

aws ecr get-login-password | docker login --username AWS --password-stdin 123456789.dkr.ecr.eu-central-1.amazonaws.com

docker push 123456789.dkr.ecr.eu-central-1.amazonaws.com/lambda-docker-etl:latest

anna-anisienia

(venv) ➜  curl -XPOST "<http://localhost:9000/2015-03-31/functions/function/invocations>" -d '{}'
"ETL finished. Old scores: [100, 100, 100, 68, 99, 45, 53, 39, 87, 99, 67, 86, 37, 100, 99]. 
New scores: [200, 200, 200, 68, 99, 45, 53, 39, 87, 99, 67, 86, 37, 200, 99]"%

anna-anisienia

prefect agent ecs start --token $RUNNER_TOKEN_FARGATE_AGENT \
 --task-role-arn=arn:aws:iam::<YOUR_ACCOUNT_NAME>:role/ECSTaskS3Role \
 --log-level INFO --label fargate-dev --label s3-flow-storage \
 --name fargate-demo > /dev/null 2>&1 &

anna-anisienia

export RUNNER_TOKEN_FARGATE_AGENT=<your token created using the command prefect auth create-token -n DemoFargateAgentToken -s RUNNER>
export AWS_ACCESS_KEY_ID=************************+
export AWS_SECRET_ACCESS_KEY=*********************
export AWS_DEFAULT_REGION=eu-central-1 = YOUR-REGION
export networkConfiguration="{'awsvpcConfiguration': {'assignPublicIp': 'ENABLED', 'subnets': ['subnet-12345', 'subnet-6789'], 'securityGroups': []}}"
export executionRoleArn=arn:aws:iam::<YOUR_ACCOUNT_ID>:role/ecsTaskExecutionRole

anna-anisienia

[ec2-user@ip-000-00-00-00 ~]$ python3
Python 3.7.9 (default, Aug 27 2020, 21:59:41)
[GCC 7.3.1 20180712 (Red Hat 7.3.1-9)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import prefect
>>> prefect.__version__
'0.14.1'
>>> import boto3
>>> boto3.__version__
'1.16.47'