ana.py

from collections import Counter
from pathlib import Path


def ngrams(n, s):
    # lazy, but trailing sub-n-grams are not statistically significant
    for i in range(len(s)):
        yield s[i:i+n]


if __name__ == "__main__":
    import sys
    _, *args = sys.argv
    if args[0].startswith("-"):
        (_, gram), *args = args
        gram = int(gram)
    else:
        gram = 2
    data = "\n".join(Path(f).read_text().lower() for f in args)
    c = Counter(ngrams(gram, data))
    top = list(reversed(sorted((v,k) for k,v in c.items())))
    total = sum(v[0] for v in top)
    for v,k in top[:20]:
        print(f"{v/total:.3f}: {k!r}")