action.yaml

name: scrape-wikipedia

on:
  push:
    branches:
      - master
  schedule:
    - cron: "0 */1 * * *"

jobs:
  build-and-deploy:
    runs-on: ubuntu-latest
    steps:
      - name: 🍽️ Get working copy
        uses: actions/checkout@master
        with:
          fetch-depth: 1                                                                 
      - name: 🐍 Set up Python 3.8                               
        uses: actions/setup-python@v2                               
        with:                                 
          python-version: '3.8'                                                   
      - name: 💿 Install Requirements                             
        run: pip install -r requirements.txt
      - name: 🍳 Update dataset
        run: python main.py
      - name: 🚀 Commit and push if it changed
        run: |
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git add -A
          timestamp=$(date -u)
          git commit -m "Latest data: ${timestamp}" || exit 0
          git push

action_1.yaml

name: scrape-wikipedia

on:
  push:
    branches:
      - master
  schedule:
    - cron: "0 */1 * * *"

action_2.yaml

jobs:
  build-and-deploy:
    runs-on: ubuntu-latest

action_3.yaml

    steps:
      - name: 🍽️ Get working copy
        uses: actions/checkout@master
        with:
          fetch-depth: 1     

action_4.yaml

      - name: 🐍 Set up Python 3.8                               
        uses: actions/setup-python@v2                               
        with:                                 
          python-version: '3.8'

action_5.yaml

      - name: 💿 Install Requirements                             
        run: pip install -r requirements.txt

action_6.yaml

      - name: 🍳 Update dataset
        run: python main.py

action_7.yaml

      - name: 🚀 Commit and push if it changed
        run: |
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git add -A
          timestamp=$(date -u)
          git commit -m "Latest data: ${timestamp}" || exit 0
          git push

main.py

import re

import pandas as pd

def scrape_wiki_table(url, table_index=0):
    df = pd.read_html(url, header=0)[table_index]
    return re.sub(r"\[?\s*(\d+)(?=(?:, \d+)|\])(?=[^\[]*\]).", "", df.to_csv(index=False))

if __name__ == '__main__':
    with open('data.csv', 'w+') as f:
        f.write(scrape_wiki_table('https://en.wikipedia.org/wiki/List_of_chief_executive_officers'))

Simple Versioned Datasets With Github Actions

# Medium Post

snippet_1.py

df = pd.read_html(url, header=0)[table_index]

snippet_2.csv

snippet_3.py

return re.sub(r"\[?\s*(\d+)(?=(?:, \d+)|\])(?=[^\[]*\]).", "", df.to_csv(index=False))