Deploying Meltano on Argo Workflows

meltano-elt-template.yml

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: meltano-elt-template
spec:
  entrypoint: run-elt
  arguments:
    parameters:
    - name: extractor_name
    - name: loader_name
    - name: meltano_repo_branch
      value: master

  templates:
  - name: run-elt
    inputs:
      parameters:
      - name: extractor_name
      - name: loader_name
      - name: meltano_repo_branch
        value: master
      artifacts:
      - name: meltano-repo
        path: /src/meltano
        git:
          repo: git@github.com:my_company/meltano-project.git
          revision: "{{inputs.parameters.meltano_repo_branch}}"
          sshPrivateKeySecret:
            name: github-credentials
            key: ssh-private-key
      - name: entrypoint
        path: /tmp/entrypoint.py
        raw:
          data: |

            # Enable access to private Github repos during meltano install
            cat << EOF > $HOME/.netrc
            machine github.com
            login ${GITHUB_USERNAME}
            password ${GITHUB_ACCESS_TOKEN}
            
            machine api.github.com
            login ${GITHUB_USERNAME}
            password ${GITHUB_ACCESS_TOKEN}
            EOF

            # prerequisites
            meltano install extractor {{inputs.parameters.extractor_name}}
            meltano install loader {{inputs.parameters.loader_name}}
            export JOB_ID='{{inputs.parameters.extractor_name}}-to-{{inputs.parameters.loader_name}}'

            # execute
            meltano elt                             \
              --job_id=$JOB_ID                      \
              {{inputs.parameters.extractor_name}}  \
              {{inputs.parameters.loader_name}}

    container:
      image: meltano/meltano:latest-python3.7
      command: ["/bin/sh", "{{inputs.artifacts.entrypoint.path}}"]
      workingDir: "{{inputs.artifacts.meltano-repo.path}}/meltano"
      env:
      - name: TAP_FOO_API_KEY
        valueFrom:
          secretKeyRef:
            name: meltano-secrets
            key: tap_foo_api_key
      - name: TAP_BAR_API_KEY
        valueFrom:
          secretKeyRef:
            name: meltano-secrets
            key: tap_bar_api_key
      - name: TARGET_SNOWFLAKE_PROD_PASSWORD
        valueFrom:
          secretKeyRef:
            name: meltano-secrets
            key: target_snowflake_prod_password
      - name: MELTANO_DATABASE_URI
        valueFrom:
          secretKeyRef:
            name: meltano-secrets
            key: meltano_database_uri
      - name: GITHUB_USERNAME
        valueFrom:
          secretKeyRef:
            name: github-credentials
            key: username
      - name: GITHUB_ACCESS_TOKEN
        valueFrom:
          secretKeyRef:
            name: github-credentials
            key: personal-access-token

tap-bar-elt-workflow.yml

apiVersion: argoproj.io/v1alpha1
kind: CronWorkflow
metadata:
  name: cron-elt-bar
spec:
  schedule: "0 0 * * *"

  workflowSpec:
    entrypoint: do

    templates:
    - name: do
      steps:
      - - name: run-elt
          templateRef:
            name: meltano-elt-template
            template: run-elt
          arguments:
            parameters:
            - name: extractor_name
              value: tap-bar
            - name: loader_name
              value: target-snowflake-prod

tap-foo-cron-workflow.yml

apiVersion: argoproj.io/v1alpha1
kind: CronWorkflow
metadata:
  name: cron-elt-foo
spec:
  schedule: "0 */6 * * *"

  workflowSpec:
    entrypoint: do

    templates:
    - name: do
      steps:
      - - name: run-elt
          templateRef:
            name: meltano-elt-template
            template: run-elt
          arguments:
            parameters:
            - name: extractor_name
              value: tap-foo
            - name: loader_name
              value: target-snowflake-prod