Example workflow

example-etl.yaml

name: ETL example
on: 
  push: 
    branches:
    - main
jobs:
  extract-transform-load:
    runs-on: ubuntu-latest
    env: 
      PG_CONN_STRING: ${{ secrets.PG_CONN_STRING }}
      BITDOTIO_USER: bitdotio
      BITDOTIO_REPO: simple_pipeline
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.7'
      - name: Install dependencies
        run: python -m pip install --upgrade pip -r requirements.txt
      - name: NYT cases by county
        run: |
             python simple_pipeline/main.py \
             -validate_data \
             -name nyt_cases_counties \
             'https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-counties.csv' \
             $BITDOTIO_USER/$BITDOTIO_REPO.cases_counties