wordcount.py

# /// script
# requires-python = ">=3.13"
# dependencies = [
#     "marimo>=0.23.3",
#     "pandas>=3.0.3",
# ]
# ///

import marimo

__generated_with = "0.23.9"
app = marimo.App()


@app.cell
def _():
    import os
    import re
    from collections import Counter
    import pandas as pd

    data_dir = ".data"
    all_words = Counter()

    # Expanded blacklist
    blacklist = {
        # HTML elements
        "span", "p", "div", "a", "br", "li", "ul", "ol", "table", "td", "tr", "th", "small", "sup", "class", "links", "js", "nbsp", "style"
        # Wiki/Technical/Functional
        "redirect", "non", "wiki", "item", "items", "via", "wikipedia", "nowiki", "category", "article",
        "user", "talk", "utc", "merged", "merge", "deleted", "closed", "done", "not", "rfd", "int", 
        "talkpagelinktext", "signature", "admin", "mediawiki", "gadget", "wikidata", "contributions", "wb", "linked", "deletion", "page", "template", "org", "en", "bot", "www",
        # Common Users
        "benebot", "deltabot", "succu", "ymblanter", "stryn", "gzwder",
        # Colors
        "color", "red", "black", "blue", "green", "yellow", "white", "orange", "purple", "brown", "grey", "gray",
        # Joining/Generic/UI/Misc
        "the", "and", "a", "to", "of", "in", "is", "it", "that", "on", "for", "with", "as", "was", "at", "by", "from", "be", "this", "are", "have", "or", "an",
        "any", "next", "these", "there", "see", "size", "also", "but", "can", "they", "you", "into", "sub", "group", "other", "only", "https", "do", "has", "same", 
        "hold", "self", "if", "request", "big", "fails", "art", "help", "mylanguage", "been", "bulk", "please", "all", "above", "use", "game", "no", "special", "font", "created",
        # Months
        "january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"
    }

    if os.path.exists(data_dir):
        for root, dirs, files in os.walk(data_dir):
            for filename in files:
                filepath = os.path.join(root, filename)
                try:
                    with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
                        text = f.read().lower()
                        words = re.findall(r'\b[a-z]{2,}\b', text)
                        all_words.update([w for w in words if w not in blacklist])
                except Exception as e:
                    print(f"Error reading {filepath}: {e}")

        word_freq_df = pd.DataFrame(all_words.most_common(1000), columns=['word', 'count'])
    else:
        word_freq_df = pd.DataFrame(columns=['word', 'count'])
        print(f"Directory {data_dir} not found.")

    word_freq_df
    return


if __name__ == "__main__":
    app.run()