model download using dotenv $envs and awscli

.env

AWS_ACCESS_KEY_ID="your_object_storage_access_key"
AWS_SECRET_ACCESS_KEY="your_object_storage_secret_key"
BUCKET_NAME="thebucket"
ENDPOINT_URL="https://endpoint"
REGION_NAME="theregion"

config.yaml

destination_dir: models/spam-sms
files:
  - exports/spam-sms/spam-model.h5
  - exports/spam-sms/spam-classifer-tokenizer.json
  - exports/spam-sms/spam-classifer-metadata.json

model-download.yaml

steps:
  - name: pypyr.steps.fetchyaml
    comment: load config values from yaml
             this way you don't need to store change-able config in the pipeline
             itself.
    in:
      fetchYaml:
        path: config.yaml

  - name: pypyr.steps.py
    comment: dotenv environment variables for local dev
             on prod these values will be in the $ENV already
    in:
      py: |
        from dotenv import load_dotenv
        load_dotenv()

  - name: pypyr.steps.env
    comment: this will raise error with $ENV name if the $ENV doesn't exist.
    in:
      env:
        get:
          aws_key: AWS_ACCESS_KEY_ID
          aws_secret: AWS_SECRET_ACCESS_KEY
          aws_region: REGION_NAME
          aws_endpoint: ENDPOINT_URL
          aws_bucket: BUCKET_NAME

  - name: pypyr.steps.pyimport
    comment: need pathlib to do download path manipulation on each file
    in:
      pyImport: from pathlib import Path

  - name: pypyr.steps.call
    comment: loop over every file in files as specified in config
    foreach: '{files}'
    in:
      call: handle_file

handle_file:
  - name: pypyr.steps.contextsetf
    comment: get local destination path for this file
    in:
      contextSetf:
        local_path: !py str(Path(destination_dir).joinpath(Path(i).name))
  
  - name: pypyr.steps.echo
    in:
      echoMe: saving s3://{aws_bucket}/{i} to {local_path}

  - name: pypyr.steps.cmd
    comment: download file from s3 to local_path using aws cli
    in:
      cmd: aws s3 cp s3://{aws_bucket}/{i} {local_path} --region {aws_region} --endpoint-url {aws_endpoint}