Last active
March 18, 2024 19:20
-
-
Save cnolanminich/6ed9317005fcf5ac8b6c06fdf23fd091 to your computer and use it in GitHub Desktop.
Use GitHub Labels to use a different AWS Account
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # modified by https://github.com/dagster-io/hooli-data-eng-pipelines/blob/christian-test-amp/.github/workflows/deploy-dagster-cloud.yml | |
| name: Dagster Cloud Hybrid Deployment | |
| on: | |
| push: # For full deployment | |
| branches: | |
| - "main" | |
| - "master" | |
| pull_request: # For branch deployments | |
| types: [opened, synchronize, reopened, closed] | |
| concurrency: | |
| # Cancel in-progress deploys to the same branch | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DAGSTER_CLOUD_ORGANIZATION: "hooli" | |
| DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| DAGSTER_PROJECT_DIR: "." | |
| DAGSTER_CLOUD_YAML_PATH: "dagster_cloud.yaml" | |
| # The IMAGE_REGISTRY should match the registry: in dagster_cloud.yaml | |
| IMAGE_REGISTRY: "764506304434.dkr.ecr.us-west-2.amazonaws.com/hooli-data-science-prod" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| dagster-cloud-deploy: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Pre-run checks | |
| id: prerun | |
| uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1.27 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: steps.prerun.outputs.result != 'skip' | |
| with: | |
| ref: ${{ github.head_ref }} | |
| - name: Validate configuration | |
| id: ci-validate | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}" | |
| - name: Initialize build session | |
| id: ci-init | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1.27 | |
| with: | |
| project_dir: ${{ env.DAGSTER_PROJECT_DIR }} | |
| dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }} | |
| deployment: 'data-eng-prod' | |
| - name: Generate docker image tag | |
| id: generate-image-tag | |
| if: steps.prerun.outputs.result != 'skip' | |
| run: echo "IMAGE_TAG=$GITHUB_SHA-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> $GITHUB_ENV && echo $IMAGE_TAG | |
| - name: Configure AWS credentials -- Deployment 1 | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| if: steps.prerun.outputs.result != 'skip' && github.event.label.name != 'deplyoment-2' | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Configure AWS credentials -- Deployment 2 | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| if: steps.prerun.outputs.result != 'skip' && github.event.label.name == 'deplyoment-2' | |
| with: | |
| aws-access-key-id: ${{ secrets.DEPLOYMENT_2_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOYMENT_2_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Login to ECR | |
| if: ${{ steps.prerun.outputs.result != 'skip' }} | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| # Copy the production manifest.json to local directory - Raise error if not retrieved | |
| - name: Download dbt manifest from S3 | |
| if: steps.prerun.outputs.result != 'skip' && github.event_name == 'pull_request' | |
| continue-on-error: true | |
| run: | | |
| output=$(aws s3 cp s3://hooli-demo/dbt_slim_ci/manifest.json ./dbt_project/target/slim_ci/manifest.json 2>&1) || echo "Error: $output" | |
| # Build 'data-eng-pipeline' code location | |
| - name: Build dbt manifest for data-eng-pipeline | |
| if: steps.prerun.outputs.result != 'skip' | |
| run: | | |
| pip install pyopenssl --upgrade; | |
| pip install click --upgrade; | |
| pip install dbt-core dbt-duckdb dbt-snowflake; | |
| make deps | |
| make manifest | |
| # Copy production manifest.json to S3 on merge | |
| - name: Upload dbt manifest to S3 | |
| if: steps.prerun.outputs.result != 'skip' && github.event_name != 'pull_request' | |
| run: | | |
| aws s3 cp ./dbt_project/target/manifest.json s3://hooli-demo/dbt_slim_ci/manifest.json | |
| - name: Build and upload Docker image for data-eng-pipeline | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-data-eng-pipeline | |
| - name: Update build session with image tag for data-eng-pipeline | |
| id: ci-set-build-output-data-eng-pipeline | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci set-build-output --location-name=data-eng-pipeline --image-tag=$IMAGE_TAG-data-eng-pipeline" | |
| # Build 'basics' code location | |
| - name: Build and upload Docker image for basics | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./hooli_basics | |
| push: true | |
| tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-basics | |
| - name: Update build session with image tag for basics | |
| id: ci-set-build-output-basics | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci set-build-output --location-name=basics --image-tag=$IMAGE_TAG-basics" | |
| # Build 'batch enrichment' code location | |
| - name: Build and upload Docker image for batch enrichment | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./hooli_batch_enrichment | |
| push: true | |
| tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-batch-enrichment | |
| - name: Update build session with image tag for batch enrichment | |
| id: ci-set-build-output-batch-enrichment | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci set-build-output --location-name=batch_enrichment --image-tag=$IMAGE_TAG-batch-enrichment" | |
| # Build 'snowflake_insights' code location | |
| - name: Build and upload Docker image for snowflake insights | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./hooli_snowflake_insights | |
| push: true | |
| tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-snowflake-insights | |
| - name: Update build session with image tag for snowflake insights | |
| id: ci-set-build-output-snowflake-insights | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci set-build-output --location-name=snowflake_insights --image-tag=$IMAGE_TAG-snowflake-insights" | |
| # Build 'demo_assets' code location | |
| - name: Build and upload Docker image for demo_assets | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./hooli-demo-assets | |
| push: true | |
| tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-demo-assets | |
| - name: Update build session with image tag for demo_assets | |
| id: ci-set-build-output-demo-assets | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci set-build-output --location-name=demo_assets --image-tag=$IMAGE_TAG-demo-assets" | |
| # Build pipes example container | |
| - name: Build and upload Docker image for pipes example | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./hooli_data_eng/utils/example_container | |
| push: true | |
| tags: ${{ env.IMAGE_REGISTRY }}:latest-pipes-example | |
| # Deploy | |
| - name: Deploy to Dagster Cloud | |
| id: ci-deploy | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci deploy" | |
| # Get branch deployment as input to job trigger below | |
| - name: Get branch deployment | |
| id: get-branch-deployment | |
| if: steps.prerun.outputs.result != 'skip' && github.event_name == 'pull_request' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/get_branch_deployment@v0.1 | |
| with: | |
| organization_id: 'hooli' | |
| # Trigger dbt slim CI job | |
| - name: Trigger dbt slim CI | |
| if: steps.prerun.outputs.result != 'skip' && github.event_name == 'pull_request' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/run@v0.1 | |
| with: | |
| location_name: data-eng-pipeline | |
| deployment: ${{ steps.get-branch-deployment.outputs.deployment }} | |
| job_name: dbt_slim_ci_job | |
| organization_id: hooli | |
| # Summary and comment updates - note these always() run | |
| - name: Update PR comment for branch deployments | |
| id: ci-notify | |
| if: steps.prerun.outputs.result != 'skip' && always() | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}" | |
| - name: Generate summary | |
| id: ci-summary | |
| if: steps.prerun.outputs.result != 'skip' && always() | |
| uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27 | |
| with: | |
| command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment