jamiekt

import datetime
from typing import Sequence
from airflow.decorators import task, dag

from airflow.providers.amazon.aws.operators.emr import EmrServerlessCreateApplicationOperator

class JamietEmrServerlessCreateApplicationOperator(EmrServerlessCreateApplicationOperator):
    template_fields: Sequence[str] = ["config"]
    
    def __init__(self, **kwargs) -> None:

jamiekt

import datetime
from airflow.decorators import task, dag

from airflow.providers.amazon.aws.operators.emr import EmrServerlessCreateApplicationOperator
    
@dag(
    dag_id="demo-xcom-problem",
    start_date=datetime.datetime(2021, 1, 1),
    catchup=False
)

jamiekt

SELECT c.borough,c.year,
    ARRAY(
      select as struct c2.month,sum(value) crime_tally
      from `bigquery-public-data.london_crime.crime_by_lsoa` as c2
      where c.borough = c2.borough
        and c.year = c2.year
      group by c2.month
    ) as monthly_summary
FROM (
  select distinct borough,year

jamiekt

return {
    match.group(0)
    for matched_policy in matched_policies
    if (match := re.search(r"p\d{6}", matched_policy)) is not None
}

jamiekt

prefixes = set()
for matched_policy in matched_policies:
    search_result = re.search(r_search, matched_policy)
    assert search_result is not None
    prefixes.add(search_result.group(0))
return prefixes

jamiekt

import re
matched_policies = {...} #some logic to derive a set
return {
        re.search(r"p\d{6}", matched_policy).group(0)
        for matched_policy in matched_policies
}

jamiekt

[tool.poetry]
name = "real-time-event-processor"
version = "0.1.0"
description = ""
authors = []
 
[tool.poetry.dependencies]
python = "^3.9, <3.10"
 
Flask = "^2.0.1"

jamiekt

➜ poetry env info

Virtualenv
Python:         3.8.2
Implementation: CPython
Path:           NA

System
Platform: darwin
OS:       posix

jamiekt

repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v4.3.0
    hooks:
      - id: end-of-file-fixer
      - id: trailing-whitespace
 
  - repo: local
    hooks:
      - id: black

jamiekt

gcloud projects get-iam-policy project-id \
  --filter="bindings.role:roles/workflows.editor" \
  --flatten="bindings[].members" \
  --format="table(bindings.members)"
  #### output
  # group:group-name@example.com
  # user:user-name@example.com
  # servceAccount:service-account-name@project-id.iam.gserviceaccount.com