import.py

# /// script
# requires-python = ">=3.13"
# dependencies = [
#     "marimo>=0.23.3",
#     "beautifulsoup4>=4.12.2",
#     "requests>=2.34.2",
#     "mwparserfromhell>=0.7.2",
#     "pandas>=3.0.3",
#     "plotly>=6.8.0",
# ]
# ///

import marimo

__generated_with = "0.23.9"
app = marimo.App()


@app.cell
def _():
    import os
    import re
    import time
    import requests
    from pathlib import Path
    from bs4 import BeautifulSoup

    BASE_URL = "https://www.wikidata.org"
    ARCHIVE_INDEX_URL = f"{BASE_URL}/wiki/Wikidata:Requests_for_deletions/Archive"
    DATA_DIR = Path(".data")

    # Set a descriptive user agent; replace with your own contact info
    USER_AGENT = (
        "WikidataRFDAnalysis/1.0 "
        "(https://www.wikidata.org/wiki/User:Addshore; "
        "mailto:addshore@addshore.com) Python-requests"
    )

    session = requests.Session()
    session.headers.update({"User-Agent": USER_AGENT})

    DATA_DIR.mkdir(exist_ok=True)

    def fetch(url: str, params: dict | None = None) -> str:
        """GET a URL with basic error handling."""
        resp = session.get(url, params=params, timeout=30)
        resp.raise_for_status()
        return resp.text


    def get_year_links() -> list[str]:
        """Parse the archive index page and return a sorted list of year strings."""
        soup = BeautifulSoup(fetch(ARCHIVE_INDEX_URL), "html.parser")
        years = set()
        for a in soup.select("#mw-content-text a"):
            m = re.match(
                r"^/wiki/Wikidata:Requests_for_deletions/Archive/(\d{4})$",
                str(a.get("href", "")),
            )
            if m:
                years.add(m.group(1))
        return sorted(years)


    def get_day_links(year: str) -> list[tuple[str, str, str]]:
        """Parse a year archive page and return a sorted list of (year, month, day)."""
        url = f"{BASE_URL}/wiki/Wikidata:Requests_for_deletions/Archive/{year}"
        soup = BeautifulSoup(fetch(url), "html.parser")
        days = set()
        for a in soup.select("#mw-content-text a"):
            m = re.match(
                r"^/wiki/Wikidata:Requests_for_deletions/Archive/(\d{4})/(\d{2})/(\d{2})$",
                str(a.get("href", "")),
            )
            if m:
                days.add((m.group(1), m.group(2), m.group(3)))
        return sorted(days)


    def fetch_wikitext(title: str) -> str:
        """Fetch the raw wikitext of a page via the MediaWiki action=raw endpoint."""
        url = f"{BASE_URL}/w/index.php"
        # Using params here ensures the title is properly URL-encoded
        return fetch(url, params={"title": title, "action": "raw"})


    years = get_year_links()

    if not years:
        print("No years found. Verify the URL or HTML structure.")
        exit(1)

    print(f"Found {len(years)} years: {years[0]}..{years[-1]}")

    for year in years:
        days = get_day_links(year)
        print(f"  {year}: {len(days)} day pages")

        for year_, month, day in days:
            # Create a directory tree .data/Archive/yyyy/mm/
            archive_dir = DATA_DIR / "Archive" / year_ / month
            archive_dir.mkdir(parents=True, exist_ok=True)
            # Store the wikitext in a file named dd.wiki inside that directory
            filepath = archive_dir / f"{day}.wiki"
            title = f"Wikidata:Requests_for_deletions/Archive/{year_}/{month}/{day}"

            if filepath.exists():
                print(f"    skip {year_}-{month}-{day}.wiki (already on disk)")
                continue

            try:
                wikitext = fetch_wikitext(title)
                filepath.write_text(wikitext, encoding="utf-8")
                size = filepath.stat().st_size
                print(f"    saved {year_}-{month}-{day}.wiki ({size:,} bytes)")
            except requests.HTTPError as e:
                print(f"    !! {year_}-{month}-{day}.wiki: {e}")
                continue

            # Be polite to the server
            time.sleep(0.25)
    return


if __name__ == "__main__":
    app.run()