CI testing

pull-request-test.yaml

name: Pull Request Test
on: 
  pull_request: 
    branches:
    - main
jobs:
  extract-transform-load:
    runs-on: ubuntu-latest
    env: 
      PG_CONN_STRING: ${{ secrets.PG_CONN_STRING }}
      BITDOTIO_USER: bitdotio
      BITDOTIO_REPO: simple_pipeline_dev
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.9.1'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip -r requirements.txt
          python -m pip install pytest
      - name: Run unit tests
        run: python -m pytest
      - name: ACS population by county
        run: |
             python simple_pipeline/main.py \
             -local_source \
             -name acs_population_counties \
             simple_pipeline/acs_5yr_population_data.csv \
             $BITDOTIO_USER/$BITDOTIO_REPO.population_counties
      - name: NYT cases by county
        run: |
             python simple_pipeline/main.py \
             -validate_data \
             -name nyt_cases_counties \
             'https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-counties.csv' \
             $BITDOTIO_USER/$BITDOTIO_REPO.cases_counties
      - name: CDC vaccines by county
        run: |
             python simple_pipeline/main.py \
             -name cdc_vaccines_counties \
             'https://data.cdc.gov/api/views/8xkx-amqh/rows.csv?accessType=DOWNLOAD' \
             $BITDOTIO_USER/$BITDOTIO_REPO.vaccinations_counties
      - name: Create CA report table
        run: |
             python simple_pipeline/sql_executor.py \
             simple_pipeline/ca_covid_data.sql \
             $BITDOTIO_USER \
             $BITDOTIO_REPO