View public-key-cryptography-demo.txt
# public-key cryptography demo for Mac or Linux | |
# create the private key - require a pass phrase | |
# openssl genrsa -aes256 -out sec-cert.pem 2048 | |
# create the private key - no pass phrase required | |
openssl genrsa -out sec-cert.pem 2048 | |
# create public key (PEM) | |
openssl rsa -in sec-cert.pem -outform PEM -pubout -out sec-cert.pub.pem |
View trigger_dag.py
#!/usr/bin/env python3 | |
# MWAA: Trigger an Apache Airflow DAG using SDK | |
# Author: Gary A. Stafford (February 2021) | |
import logging | |
import boto3 | |
import requests |
View ssh_spark_submit.sh
ssh -i ~/.ssh/${EC2_KEY_NAME}.pem hadoop@${EMR_MASTER_DNS} \ | |
""" | |
spark-submit --deploy-mode cluster --master yarn \ | |
--conf spark.yarn.submit.waitAppCompletion=true \ | |
's3a://${WORK_BUCKET}/analyze/bakery_sales_ssm.py' | |
""" |
View ssh_spark_submit.sh
ssh -i ~/.ssh/${EC2_KEY_NAME}.pem hadoop@${EMR_MASTER_DNS} \ | |
""" | |
spark-submit --deploy-mode cluster --master yarn \ | |
--conf spark.yarn.submit.waitAppCompletion=true \ | |
's3a://${WORK_BUCKET}/analyze/bakery_sales_ssm.py' | |
""" |
View mwaa_configuration.json
{ | |
"core.default_ui_timezone": "America/Los_Angeles" | |
} |
View mwaa_get_environment.sh
aws mwaa get-environment \ | |
--name <your_environment_name> | \ | |
jq -r '.Environment.AirflowConfigurationOptions' |
View mwaa_update_environment.sh
aws mwaa update-environment \ | |
--name <your_environment_name> \ | |
--airflow-configuration-options """{ | |
\"core.default_ui_timezone\": \"America/Los_Angeles\" | |
}""" |
View get_py_pkgs.py
list_python_packages_operator = BashOperator( | |
task_id='list_python_packages', | |
bash_command='python3 -m pip list' | |
) |
View mwaa_python_packages.txt
[2020-12-26 21:53:06,310] {{bash_operator.py:136}} INFO - Temporary script location: /tmp/airflowtmp2whgp_p8/list_python_packagesxo8slhc6 | |
[2020-12-26 21:53:06,350] {{bash_operator.py:146}} INFO - Running command: python3 -m pip list | |
[2020-12-26 21:53:06,395] {{bash_operator.py:153}} INFO - Output: | |
[2020-12-26 21:53:06,750] {{bash_operator.py:157}} INFO - Package Version | |
[2020-12-26 21:53:06,786] {{bash_operator.py:157}} INFO - ---------------------- --------- | |
[2020-12-26 21:53:06,815] {{bash_operator.py:157}} INFO - alembic 1.4.2 | |
[2020-12-26 21:53:06,856] {{bash_operator.py:157}} INFO - amqp 2.6.1 | |
[2020-12-26 21:53:06,898] {{bash_operator.py:157}} INFO - apache-airflow 1.10.12 | |
[2020-12-26 21:53:06,929] {{bash_operator.py:157}} INFO - apispec 1.3.3 | |
[2020-12-26 21:53:06,960] {{bash_operator.py:157}} INFO - argcomplete 1.12.0 |
View airflow_env_vars_updatted.txt
[2020-12-26 05:00:57,756] {{standard_task_runner.py:78}} INFO - Job 293: Subtask get_env_vars_task | |
[2020-12-26 05:00:58,158] {{logging_mixin.py:112}} INFO - 'AIRFLOW_CONFIG_SECRETS': '{"AIRFLOW__CORE__DEFAULT_UI_TIMEZONE":"America/New_York"}' | |
[2020-12-26 05:00:58,190] {{logging_mixin.py:112}} INFO - 'AIRFLOW_CONN_AWS_DEFAULT': 'aws://' | |
... | |
[2020-12-26 05:01:00,537] {{logging_mixin.py:112}} INFO - 'AIRFLOW__CORE__DAG_CONCURRENCY': '10000' | |
[2020-12-26 05:01:00,578] {{logging_mixin.py:112}} INFO - 'AIRFLOW__CORE__DEFAULT_UI_TIMEZONE': 'America/New_York' | |
[2020-12-26 05:01:00,630] {{logging_mixin.py:112}} INFO - 'AIRFLOW__CORE__EXECUTOR': 'CeleryExecutor' | |
... |
NewerOlder