Setting up a daily status code checker

daily_status_code.py

import datetime

import advertools as adv

today = datetime.datetime.now(datetime.UTC).strftime("%Y_%m_%d")

sitemap = adv.sitemap_to_df("https://example/sitemap.xml")

adv.crawl_headers(
    sitemap["loc"],
    f"path/to/status_codes/{today}.jl",
    custom_settings={
        "AUTOTHROTTLE_ENABLED": False,
        "LOG_FILE": f"path/to/status_codes_logs/{today}.jl",
    },
)

Add this chron task. You can change @daily to @Weekly or @monthly

crontab -e

# Add this line to the end of the file, making sure to replace /path/python_virtualenv/ with your path

@daily PATH=/path/python_virtualenv/venv/bin/ ; /path/python_virtualenv/venv/bin/python /path/to/daily_status_codes.py