Skip to content

Instantly share code, notes, and snippets.

Avatar

Anna Geller anna-anisienia

View GitHub Profile
View Dockerfile
# set a base image that includes Lambda Runtime API:
# Source: https://hub.docker.com/r/amazon/aws-lambda-python
FROM amazon/aws-lambda-python:3.8
# optional: ensure that pip is up to date
RUN /var/lang/bin/python3.8 -m pip install --upgrade pip
# first we COPY only requirements.txt to ensure that later builds
# with changes to your src code will be faster due to caching of this layer
COPY requirements.txt .
View trigger_and_wait_for_completion.py
from airflow.models import DagModel, TaskInstance, DagRun
from airflow.utils.decorators import apply_defaults
from airflow.operators.dagrun_operator import TriggerDagRunOperator
from airflow.sensors.base_sensor_operator import BaseSensorOperator
from airflow.utils.db import provide_session
class WaitForCompletion(BaseSensorOperator):
"""
Waits for a different DAG or a task in a different DAG to complete
View local_slack_dq_alerts.py
import json
import logging
import requests
import pandas as pd
import awswrangler as wr
from typing import List, Any
class DataQualityAlert:
def __init__(self, slack_webhook_url: str, database: str = "ecommerce"):
View sqs.py
import boto3
import uuid
import json
import random
import logging
from datetime import datetime
logging.basicConfig(format="[%(levelname)s] [%(name)s] [%(asctime)s]: %(message)s",
level="INFO")
logger = logging.getLogger(__name__)
View sns.py
import boto3
import uuid
import json
import random
import logging
from datetime import datetime
logging.basicConfig(format="[%(levelname)s] [%(name)s] [%(asctime)s]: %(message)s",
level="INFO")
logger = logging.getLogger(__name__)
View kinesis_data_streams.py
import requests
import boto3
import uuid
import time
import json
client = boto3.client('kinesis', region_name='eu-central-1')
partition_key = str(uuid.uuid4())
View Dockerfile
# set a base image: https://hub.docker.com/_/python?tab=tags&page=1&ordering=last_updated
FROM python:3.8
# optional: ensure that pip is up to date
RUN pip install --upgrade pip
# first we COPY only requirements.txt to ensure that later builds
# with changes to your src code will be faster due to caching of this layer
COPY requirements.txt .
RUN pip install -r requirements.txt
View timestream_while_loop.py
import time
import requests
import logging
import boto3
from botocore.config import Config
def send_price_data_to_timestream(write_client):
base_url = "https://min-api.cryptocompare.com/data"
symbols = "BTC,ETH,REP,DASH"
View timestream_1min_schedule.py
import time
import requests
import logging
import boto3
from botocore.config import Config
def send_price_data_to_timestream(write_client):
base_url = "https://min-api.cryptocompare.com/data"
symbols = "BTC,ETH,REP,DASH"
View kinesis_data_streams.py
import requests
import boto3
import uuid
import time
import json
client = boto3.client('kinesis', region_name='eu-central-1')
partition_key = str(uuid.uuid4())