dbt Workflow Template for Argo

dbt-execute-template.yml

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: dbt-exec-template
spec:
  entrypoint: dbt-execute

  arguments:
    parameters:
    - name: exec_command
      value: run
    - name: exec_command_args
      value: "-h"

  templates:

  - name: dbt-execute
    inputs:
      parameters:
      - name: exec_command
      - name: exec_command_args
      artifacts:
      - name: dbt-project
        path: /src
        git:
          repo: git@github.com:my_company/dbt-repo.git
          revision: main
      - name: dbt-profile
        path: /src/profiles.yml
        raw:
          data: |
            warehouse:
              target: dbt
              outputs:
                dbt:
                  type: snowflake
                  account: foo-bar.us-east-1
                  port: 443
                  user: dbt_user
                  password: "{{ env_var('DBT_USER_PASSWORD') }}"
                  role: dbt_role
                  warehouse: dbt_wh
                  database: analytics
                  schema: dbt
                  threads: 4
      - name: entrypoint
        path: /tmp/run_dbt.sh
        raw:
          data: |
            # install debian packages
            apt-get update && apt-get install -y g++ pkg-config libicu-dev

            # install python packages
            pip install dbt --quiet

            # install deps and run commands
            dbt deps --profiles-dir /src
            dbt {{inputs.parameters.exec_command}} {{inputs.parameters.exec_command_args}} --profiles-dir /src
    container:
      image: python:3.8-slim
      workingDir: "{{inputs.artifacts.dbt-project.path}}"
      command: ["/bin/sh", "{{inputs.artifacts.entrypoint.path}}"]
      env:
      - name: DBT_USER_PASSWORD
        valueFrom:
          secretKeyRef:
            name: dbt-credentials
            key: password

dbt-run-production-models-cron.yml

apiVersion: argoproj.io/v1alpha1
kind: CronWorkflow
metadata:
  name: cron-dbt-run-production-models
spec:
  schedule: "0 */1 * * *"

  workflowSpec:
    entrypoint: do

    templates:
    - name: do
      steps:
      - - name: run-dbt
          templateRef:
            name: dbt-exec-template
            template: dbt-execute
          arguments:
            parameters:
            - name: exec_command
              value: run
            - name: exec_command_args
              value: -m tag:production