scrapper_workflow.yml

name: Modi Speech Scrapper
on:
  schedule:
    # Runs every day at 8 AM
    - cron: '0 8 * * *'
  # Run workflow manually (without waiting for the cron to be called), through the Github Actions Workflow page directly
  workflow_dispatch:

jobs:
  modi-speech-scrapper:
    name: Delta updater
    runs-on: ubuntu-latest
    # Not needed if secrets are repo secrets. Needed only if secrets are created under an environment
    environment: prod

    steps:
      # Step 1: Checkout our code to the VM
      - name: checkout repo content
        uses: actions/checkout@v2 # checkout the repository content to github runner.
      
      # Step 2: Setup python version that we need
      - name: setup python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      
      # Step 3: Install virtual environment. In this case Poetry
      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true
          
      # Step 4: Cache the poetry dependencies for the next run
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
          
      # Step 5: Install poetry dependencies
      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root --no-dev
      
      # Step 6: Install libraries if any
      - name: Install library
        run: poetry install --no-interaction --no-dev
        
      # Step 7: Activate the venv and run the scrapper.py file.
      # Change this filename to your scrapper code
      - name: Run tests
        run: |
          source .venv/bin/activate
          python scrapper.py
        
        # Kaggle credentials as environment variables
        env:
          KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
          KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}