Count repeated tool output in Claude Code logs — byte-identical repeats + slice re-reads (containment)

count_repeated_tool_output.py

#!/usr/bin/env python3
"""Count repeated tool output in Claude Code logs.

Walks ~/.claude/projects/**/*.jsonl, resets the "seen" set at each compact_boundary
(so per-session counts don't leak across compactions), and reports:
  - byte-identical repeats  (exact-hash dedup)
  - slice re-reads          (a Read whose line-range is contained in a range
                             already read in full this session — hash & Jaccard
                             both miss these; containment catches them)

Edit/Write results are excluded: their confirmation text is identical boilerplate
and inflates the exact-repeat count several-fold.

Usage:  python3 count_repeated_tool_output.py
"""
import glob, json, os, re, hashlib
from collections import defaultdict

results = exact = 0
rchars = dup_chars = 0
reads = slice_reads = 0
read_chars = slice_chars = 0

RANGE = re.compile(r"offset[=:]\s*(\d+).*?limit[=:]\s*(\d+)", re.I | re.S)

def block_text(content):
    if isinstance(content, str):
        return content
    if isinstance(content, list):
        return " ".join(c.get("text", "") for c in content if isinstance(c, dict))
    return ""

for f in glob.glob(os.path.expanduser("~/.claude/projects/**/*.jsonl"), recursive=True):
    seen = set()                       # result hashes seen this session
    full = defaultdict(list)           # path -> list of (start, end) read in full
    tool_by_id = {}                    # tool_use_id -> (name, input) from assistant turns
    for line in open(f, errors="ignore"):
        try:
            ev = json.loads(line)
        except Exception:
            continue
        if ev.get("type") == "compact_boundary" or ev.get("subtype") == "compact_boundary":
            seen.clear(); full.clear(); continue
        msg = ev.get("message") or {}
        content = msg.get("content") or []
        if not isinstance(content, list):
            continue
        # index tool_use calls so we can tie a result back to its Read args
        for b in content:
            if isinstance(b, dict) and b.get("type") == "tool_use":
                tool_by_id[b.get("id")] = (b.get("name", ""), b.get("input") or {})
        for b in content:
            if not isinstance(b, dict) or b.get("type") != "tool_result":
                continue
            name, inp = tool_by_id.get(b.get("tool_use_id"), ("", {}))
            if name in ("Edit", "Write", "MultiEdit"):
                continue
            text = block_text(b.get("content"))
            if len(text) < 200:
                continue
            results += 1; rchars += len(text)
            h = hashlib.sha1(text.encode()).hexdigest()
            if h in seen:
                exact += 1; dup_chars += len(text)
            seen.add(h)
            # slice-reread detection for Read
            if name == "Read":
                reads += 1; read_chars += len(text)
                path = (inp.get("file_path") or "").strip()
                off = inp.get("offset"); lim = inp.get("limit")
                if path and off is not None and lim is not None:
                    start, end = int(off), int(off) + int(lim)
                    if any(s <= start and end <= e for s, e in full.get(path, [])):
                        slice_reads += 1; slice_chars += len(text)
                elif path:
                    full[path].append((0, 10**9))    # full read

pct = lambda a, b: (a / b * 100) if b else 0.0
print(f"tool results >200 chars : {results}")
print(f"byte-identical repeats  : {exact}  ({pct(exact, results):.2f}%),  "
      f"{pct(dup_chars, rchars):.2f}% of result chars")
print(f"file reads              : {reads}")
print(f"slice re-reads          : {slice_reads}  ({pct(slice_reads, reads):.1f}% of reads, "
      f"{pct(slice_chars, read_chars):.1f}% of read chars)")