Skip to content

Instantly share code, notes, and snippets.

#!/usr/bin/env bash
set -Eeuo pipefail
###################################################################################################
# Lake Formation 向け IAM ユーザーの作成 or 削除
#
# 下記ページに記載されているユーザー種類毎に、必要となる権限を付与した IAM ユーザーを作成する
# Lake Formation personas and IAM permissions reference - Personas suggested permissions
# https://docs.aws.amazon.com/lake-formation/latest/dg/permissions-reference.html#lf-permissions-tables
#
#!/usr/bin/env bash
set -Eeuo pipefail
trap finally SIGINT SIGTERM ERR EXIT
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
readonly SCRIPT_DIR
function usage() {
cat <<-EOF
Usage: $(basename "${BASH_SOURCE[0]}") [-h] [-v] [-p param_value] arg1 [arg2]
@goodbyegangster
goodbyegangster / publisher.py
Created October 19, 2022 22:17
Cloud Pub/Sub Publisher Script
import argparse
import dataclasses
import datetime
import json
import random
import time
import uuid
from collections.abc import Generator
from google.cloud import pubsub
import pytest
import pprint
from airflow.models import DagBag
class TestDag:
@pytest.fixture()
def dagbag(self):
return DagBag()
from datetime import datetime, timedelta
from textwrap import dedent
from airflow import DAG
from airflow.decorators import task
# define default arguments for operators.
default_args = {
"retries": 1,
"retry_delay": timedelta(minutes=5),
@goodbyegangster
goodbyegangster / dag_bq_sample.py
Created July 1, 2022 05:20
A Sample Google Provider DAG
from datetime import datetime
from airflow import models
from airflow.providers.google.cloud.operators.bigquery import (
BigQueryCreateEmptyDatasetOperator,
BigQueryCreateEmptyTableOperator,
BigQueryInsertJobOperator,
)
CONNECTION_ID = "conn_sample_gcp"
@goodbyegangster
goodbyegangster / sample_external_task_sensor.py
Created June 27, 2022 02:16
A Sample ExternalTaskSensor DAG
from datetime import datetime, timedelta, timezone
from airflow import DAG
from airflow.operators.empty import EmptyOperator
from airflow.sensors.external_task import (
ExternalTaskMarker,
ExternalTaskSensor,
)
JST = timezone(timedelta(hours=+9), 'JST')
@goodbyegangster
goodbyegangster / trigger_target_dag.py
Created June 26, 2022 06:13
A Sample Trigger Target DAG
from datetime import datetime
from airflow import DAG
from airflow.decorators import task
@task(task_id='pring_message')
def print_message(dag_run=None) -> None:
print(f'received message: {dag_run.conf["message"]}.')
@goodbyegangster
goodbyegangster / trigger_run_dag.py
Created June 26, 2022 06:12
A Sample TriggerDagRunOperator DAG
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.trigger_dagrun import TriggerDagRunOperator
with DAG(
dag_id='trigger_run_dag',
description='A Sample TriggerDagRunOperator DAG',
schedule_interval=timedelta(days=1),
start_date=datetime(2022, 6, 1),
@goodbyegangster
goodbyegangster / sample_sco_dag.py
Last active June 26, 2022 02:15
Apache Airflow Sample ShortCircuitOperator DAG
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.python import ShortCircuitOperator
from airflow.operators.empty import EmptyOperator
def is_even_number() -> bool:
import random
number = random.randint(1, 10)