garystafford

# public-key cryptography demo for Mac or Linux

# create the private key - require a pass phrase
# openssl genrsa -aes256 -out sec-cert.pem 2048

# create the private key - no pass phrase required
openssl genrsa -out sec-cert.pem 2048

# create public key (PEM)
openssl rsa -in sec-cert.pem -outform PEM -pubout -out sec-cert.pub.pem

garystafford

#!/usr/bin/env python3

# MWAA: Trigger an Apache Airflow DAG using SDK
# Author: Gary A. Stafford (February 2021)

import logging

import boto3
import requests

garystafford

ssh -i ~/.ssh/${EC2_KEY_NAME}.pem hadoop@${EMR_MASTER_DNS} \
"""
  spark-submit --deploy-mode cluster --master yarn \
  --conf spark.yarn.submit.waitAppCompletion=true \
  's3a://${WORK_BUCKET}/analyze/bakery_sales_ssm.py'
"""

garystafford

ssh -i ~/.ssh/${EC2_KEY_NAME}.pem hadoop@${EMR_MASTER_DNS} \
"""
  spark-submit --deploy-mode cluster --master yarn \
  --conf spark.yarn.submit.waitAppCompletion=true \
  's3a://${WORK_BUCKET}/analyze/bakery_sales_ssm.py'
"""

garystafford

{
  "core.default_ui_timezone": "America/Los_Angeles"
}

garystafford

aws mwaa get-environment \
    --name <your_environment_name> | \
    jq -r '.Environment.AirflowConfigurationOptions'

garystafford

aws mwaa update-environment \
    --name <your_environment_name> \
    --airflow-configuration-options """{
        \"core.default_ui_timezone\": \"America/Los_Angeles\"
    }"""

garystafford

list_python_packages_operator = BashOperator(
    task_id='list_python_packages',
    bash_command='python3 -m pip list'
)

garystafford

[2020-12-26 21:53:06,310] {{bash_operator.py:136}} INFO - Temporary script location: /tmp/airflowtmp2whgp_p8/list_python_packagesxo8slhc6
[2020-12-26 21:53:06,350] {{bash_operator.py:146}} INFO - Running command: python3 -m pip list
[2020-12-26 21:53:06,395] {{bash_operator.py:153}} INFO - Output:
[2020-12-26 21:53:06,750] {{bash_operator.py:157}} INFO - Package                Version
[2020-12-26 21:53:06,786] {{bash_operator.py:157}} INFO - ---------------------- ---------
[2020-12-26 21:53:06,815] {{bash_operator.py:157}} INFO - alembic                1.4.2
[2020-12-26 21:53:06,856] {{bash_operator.py:157}} INFO - amqp                   2.6.1
[2020-12-26 21:53:06,898] {{bash_operator.py:157}} INFO - apache-airflow         1.10.12
[2020-12-26 21:53:06,929] {{bash_operator.py:157}} INFO - apispec                1.3.3
[2020-12-26 21:53:06,960] {{bash_operator.py:157}} INFO - argcomplete            1.12.0

garystafford

[2020-12-26 05:00:57,756] {{standard_task_runner.py:78}} INFO - Job 293: Subtask get_env_vars_task
[2020-12-26 05:00:58,158] {{logging_mixin.py:112}} INFO - 'AIRFLOW_CONFIG_SECRETS': '{"AIRFLOW__CORE__DEFAULT_UI_TIMEZONE":"America/New_York"}'
[2020-12-26 05:00:58,190] {{logging_mixin.py:112}} INFO - 'AIRFLOW_CONN_AWS_DEFAULT': 'aws://'
...
[2020-12-26 05:01:00,537] {{logging_mixin.py:112}} INFO - 'AIRFLOW__CORE__DAG_CONCURRENCY': '10000'
[2020-12-26 05:01:00,578] {{logging_mixin.py:112}} INFO - 'AIRFLOW__CORE__DEFAULT_UI_TIMEZONE': 'America/New_York'
[2020-12-26 05:01:00,630] {{logging_mixin.py:112}} INFO - 'AIRFLOW__CORE__EXECUTOR': 'CeleryExecutor'
...