use configurable pypyr taskrunner to download files from s3

config.yaml

AWS_ACCESS_KEY_ID: mykeyID
AWS_SECRET_ACCESS_KEY: mysecretkey
endpoint_url: myendpoint
region: myregion
bucket: mybucket
destination_dir: models/spam-sms
files:
  - exports/spam-sms/spam-model.h5
  - exports/spam-sms/spam-classifer-tokenizer.json
  - exports/spam-sms/spam-classifer-metadata.json

model-download.yaml

steps:
  - name: pypyr.steps.fetchyaml
    comment: load config values from yaml
             this way you don't need to store change-able config in the pipeline
             itself.
    in:
      fetchYaml:
        path: config.yaml
  
  - name: pypyr.steps.assert
    comment: assert mandatory values specified
    foreach:
      - '{AWS_ACCESS_KEY_ID}'
      - '{AWS_SECRET_ACCESS_KEY}'
      - '{bucket}'
      - '{region}'
    in:
      assert: !py bool(i)

  - name: pypyr.steps.pyimport
    comment: need pathlib to do download path manipulation on each file
    in:
      pyImport: from pathlib import Path

  - name: pypyr.steps.call
    comment: loop over every file in files as specified in config
    foreach: '{files}'
    in:
      call: handle_file


handle_file:
  - name: pypyr.steps.echo
    in:
      echoMe: downloading {i}...

  - name: pypyr.steps.contextsetf
    comment: get local destination path for this file
    in:
      contextSetf:
        local_path: !py str(Path(destination_dir).joinpath(Path(i).name))
  
  - name: pypyr.steps.echo
    in:
      echoMe: saving s3://{bucket}/{i} to {local_path}

  - name: pypyraws.steps.client
    comment: download file from s3 to local_path location
    in:
      awsClientIn:
        serviceName: s3
        clientArgs:
          aws_access_key_id: '{AWS_ACCESS_KEY_ID}'
          aws_secret_access_key: '{AWS_SECRET_ACCESS_KEY}'
          region_name: '{region}'
          endpoint_url: '{endpoint_url}'
        methodName: download_file
        methodArgs:
          Bucket: '{bucket}'
          Key: '{i}'
          Filename: '{local_path}'