This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -Eeuo pipefail | |
################################################################################################### | |
# Lake Formation 向け IAM ユーザーの作成 or 削除 | |
# | |
# 下記ページに記載されているユーザー種類毎に、必要となる権限を付与した IAM ユーザーを作成する | |
# Lake Formation personas and IAM permissions reference - Personas suggested permissions | |
# https://docs.aws.amazon.com/lake-formation/latest/dg/permissions-reference.html#lf-permissions-tables | |
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -Eeuo pipefail | |
trap finally SIGINT SIGTERM ERR EXIT | |
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) | |
readonly SCRIPT_DIR | |
function usage() { | |
cat <<-EOF | |
Usage: $(basename "${BASH_SOURCE[0]}") [-h] [-v] [-p param_value] arg1 [arg2] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse | |
import dataclasses | |
import datetime | |
import json | |
import random | |
import time | |
import uuid | |
from collections.abc import Generator | |
from google.cloud import pubsub |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pytest | |
import pprint | |
from airflow.models import DagBag | |
class TestDag: | |
@pytest.fixture() | |
def dagbag(self): | |
return DagBag() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime, timedelta | |
from textwrap import dedent | |
from airflow import DAG | |
from airflow.decorators import task | |
# define default arguments for operators. | |
default_args = { | |
"retries": 1, | |
"retry_delay": timedelta(minutes=5), |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime | |
from airflow import models | |
from airflow.providers.google.cloud.operators.bigquery import ( | |
BigQueryCreateEmptyDatasetOperator, | |
BigQueryCreateEmptyTableOperator, | |
BigQueryInsertJobOperator, | |
) | |
CONNECTION_ID = "conn_sample_gcp" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime, timedelta, timezone | |
from airflow import DAG | |
from airflow.operators.empty import EmptyOperator | |
from airflow.sensors.external_task import ( | |
ExternalTaskMarker, | |
ExternalTaskSensor, | |
) | |
JST = timezone(timedelta(hours=+9), 'JST') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime | |
from airflow import DAG | |
from airflow.decorators import task | |
@task(task_id='pring_message') | |
def print_message(dag_run=None) -> None: | |
print(f'received message: {dag_run.conf["message"]}.') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime, timedelta | |
from airflow import DAG | |
from airflow.operators.trigger_dagrun import TriggerDagRunOperator | |
with DAG( | |
dag_id='trigger_run_dag', | |
description='A Sample TriggerDagRunOperator DAG', | |
schedule_interval=timedelta(days=1), | |
start_date=datetime(2022, 6, 1), |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime, timedelta | |
from airflow import DAG | |
from airflow.operators.python import ShortCircuitOperator | |
from airflow.operators.empty import EmptyOperator | |
def is_even_number() -> bool: | |
import random | |
number = random.randint(1, 10) |
NewerOlder