Skip to content

Instantly share code, notes, and snippets.

Avatar

Anna Geller anna-anisienia

View GitHub Profile
View aws_config_query.sql
SELECT
resourceId,
resourceType,
awsRegion,
resourceCreationTime,
tags,
tags.tag,
configuration.state.value
WHERE
resourceType NOT IN ('AWS::EC2::SecurityGroup',
View good_data_api_first_steps.py
import requests
from pprint import pprint
base_url = "http://localhost:3000/api"
auth = {"Authorization": "Bearer YWRtaW46Ym9vdHN0cmFwOmFkbWluMTIz"}
workspaces = requests.get(f"{base_url}/entities/workspaces", headers=auth)
data = workspaces.json().get("data")
workspace_id = data[0].get("id")
dashboards = requests.get(f"{base_url}/entities/workspaces/{workspace_id}/analyticalDashboards",
View dynamodb_example_powerplant_dataset.py
"""
Dataset used: https://datasets.wri.org/dataset/globalpowerplantdatabase
"""
import os
import time
import pandas as pd
from datetime import datetime, timedelta
import awswrangler as wr
from boto3.dynamodb.conditions import Key
View read_large_file.py
"""
Dataset used: https://datasets.wri.org/dataset/globalpowerplantdatabase
"""
import os
import time
import pandas as pd
from datetime import datetime, timedelta
import awswrangler as wr
from boto3.dynamodb.conditions import Key
View ex_taskflow_and_task_group.py
import os
import boto3
import tempfile
from airflow import DAG
from airflow.decorators import task
from airflow.utils.dates import days_ago
from airflow.providers.amazon.aws.transfers.s3_to_redshift import S3ToRedshiftOperator
from airflow.utils.task_group import TaskGroup
S3_BUCKET = 'test_bucket_123'
View ecr.bash
aws ecr create-repository --repository-name lambda-docker-etl --image-scanning-configuration scanOnPush=true
docker tag lambda-docker-etl:latest 123456789.dkr.ecr.eu-central-1.amazonaws.com/lambda-docker-etl:latest
aws ecr get-login-password | docker login --username AWS --password-stdin 123456789.dkr.ecr.eu-central-1.amazonaws.com
docker push 123456789.dkr.ecr.eu-central-1.amazonaws.com/lambda-docker-etl:latest
View tmp.bash
(venv) ➜ curl -XPOST "<http://localhost:9000/2015-03-31/functions/function/invocations>" -d '{}'
"ETL finished. Old scores: [100, 100, 100, 68, 99, 45, 53, 39, 87, 99, 67, 86, 37, 100, 99].
New scores: [200, 200, 200, 68, 99, 45, 53, 39, 87, 99, 67, 86, 37, 200, 99]"%
View prefect_agent.bash
prefect agent ecs start --token $RUNNER_TOKEN_FARGATE_AGENT \
--task-role-arn=arn:aws:iam::<YOUR_ACCOUNT_NAME>:role/ECSTaskS3Role \
--log-level INFO --label fargate-dev --label s3-flow-storage \
--name fargate-demo > /dev/null 2>&1 &
View set_env_vars.bash
export RUNNER_TOKEN_FARGATE_AGENT=<your token created using the command prefect auth create-token -n DemoFargateAgentToken -s RUNNER>
export AWS_ACCESS_KEY_ID=************************+
export AWS_SECRET_ACCESS_KEY=*********************
export AWS_DEFAULT_REGION=eu-central-1 = YOUR-REGION
export networkConfiguration="{'awsvpcConfiguration': {'assignPublicIp': 'ENABLED', 'subnets': ['subnet-12345', 'subnet-6789'], 'securityGroups': []}}"
export executionRoleArn=arn:aws:iam::<YOUR_ACCOUNT_ID>:role/ecsTaskExecutionRole
View ec2_check.bash
[ec2-user@ip-000-00-00-00 ~]$ python3
Python 3.7.9 (default, Aug 27 2020, 21:59:41)
[GCC 7.3.1 20180712 (Red Hat 7.3.1-9)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import prefect
>>> prefect.__version__
'0.14.1'
>>> import boto3
>>> boto3.__version__
'1.16.47'