| class XComEnabledAWSAthenaOperator(AWSAthenaOperator): | |
| def execute(self, context): | |
| super(XComEnabledAWSAthenaOperator, self).execute(context) | |
| # just so that this gets `xcom_push`(ed) | |
| return self.query_execution_id |
| export AIRFLOW_CONN_AWS_DEFAULT="s3://$AWS_CLIENT_ID:$AWS_CLIENT_SECRET@my-bucket?region_name=$AWS_REGION" |
| from airflow.models import DAG | |
| from airflow.operators.s3_file_transform_operator import S3FileTransformOperator | |
| from datetime import datetime | |
| with DAG(dag_id='simple_s3_copy', | |
| schedule_interval=None, | |
| start_date=datetime(2019, 6, 7)) as dag: | |
| copy_file_over = S3FileTransformOperator( | |
| task_id='copy_file_over', |
| from airflow.models import DAG | |
| from airflow.contrib.operators.aws_athena_operator import AWSAthenaOperator | |
| from datetime import datetime | |
| with DAG(dag_id='simple_athena_query', | |
| schedule_interval=None, | |
| start_date=datetime(2019, 5, 21)) as dag: | |
| run_query = AWSAthenaOperator( | |
| task_id='run_query', |
| from airflow.models import DAG | |
| from airflow.contrib.operators.aws_athena_operator import AWSAthenaOperator | |
| from datetime import datetime | |
| with DAG(dag_id='simple_athena_query', | |
| schedule_interval=None, | |
| start_date=datetime(2019, 5, 21)) as dag: | |
| run_query = AWSAthenaOperator( | |
| task_id='run_query', |
| import pkg_resources | |
| installed_packages = pkg_resources.working_set | |
| for package in sorted(installed_packages, lambda x: x.key): | |
| print "{}=={}".format(package.key, package.version) |
| awscli==1.14.45 | |
| boto3==1.5.35 | |
| botocore==1.8.49 | |
| colorama==0.3.7 | |
| docutils==0.14 | |
| futures==3.2.0 | |
| jmespath==0.9.3 | |
| numpy==1.14.0 | |
| pandas==0.22.0 | |
| pip==9.0.1 |
What options are there for connecting to the warehouse? Eg: ODBC, a python module, etc
Is there a consensus around which of the options in #1 is the most featureful/best supported/most mature/etc option?
PyAthena, as it does not have any other non-pythonic requirements.
| # Example xorg.conf.d snippet that assigns the touchpad driver | |
| # to all touchpads. See xorg.conf.d(5) for more information on | |
| # InputClass. | |
| # DO NOT EDIT THIS FILE, your distribution will likely overwrite | |
| # it when updating. Copy (and rename) this file into | |
| # /etc/X11/xorg.conf.d first. | |
| # Additional options may be added in the form of | |
| # Option "OptionName" "value" | |
| # | |
| Section "InputClass" |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <script> | |
| var player = document.getElementById("player"); | |
| player.play(); | |
| </script> | |
| </head> | |
| <body> | |
| <audio id="player"> |