Local Coding Agent Usage Counter

token-counter.ts

/* USAGE:
  node scripts/sync-token-usage.mjs            # merges new days into data/token-usage.json
  node scripts/sync-token-usage.mjs --dry-run  # prints the summary, writes nothing

  Reads local token logs from supported coding agents, totals the tokens against
  known model pricing, and keeps a running daily history in data/token-usage.json.
  The /token-usage page reads that file. No database: usage is derived from local
  token counts, not billed through an API.

  Providers:
    - Claude Code: JSONL files under ~/.claude/projects, cost estimated from token logs.
    - Codex: JSONL files under ~/.codex/sessions, tokens counted from token_count events.
      Codex cost is estimated with OpenAI API Standard short-context pricing for
      GPT models. ChatGPT-authenticated Codex may not bill this way exactly, but
      this keeps the public page comparable across providers.

  ACCUMULATES. Claude Code only keeps local transcripts for ~30 days
  (cleanupPeriodDays, default 30), so the logs are a rolling window. Each run
  recomputes the days still in the logs and MERGES them into the days already
  saved, keeping older days whose logs have since been pruned. A day's record
  never regresses: on merge we keep whichever copy has more messages, so a
  partially-pruned day can't shrink the history. The file is the source of truth
  for everything older than the window, so don't delete it.
*/

import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import readline from "node:readline";
import { fileURLToPath } from "node:url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, "..");
const CLAUDE_LOGS_DIR = path.join(os.homedir(), ".claude", "projects");
const CODEX_LOGS_DIR = path.join(os.homedir(), ".codex", "sessions");
const OUT_FILE = path.join(REPO_ROOT, "data", "token-usage.json");
const MIN_USAGE_DATE = "2026-05-04";
const MIN_USAGE_MS = Date.parse(`${MIN_USAGE_DATE}T00:00:00Z`);

const DRY_RUN = process.argv.includes("--dry-run");

// USD per 1M tokens. Cache rates are derived from the input rate below.
// Matched by substring so date-suffixed and minor-version ids all resolve.
const CLAUDE_FAMILY_RATES = [
  { match: "fable", input: 10, output: 50 },
  { match: "opus", input: 5, output: 25 },
  { match: "sonnet", input: 3, output: 15 },
  { match: "haiku", input: 1, output: 5 },
];

// OpenAI API Standard, short-context USD per 1M tokens.
// Keep longer names before base models so gpt-5.4-mini does not match gpt-5.4.
const OPENAI_STANDARD_RATES = [
  { match: "gpt-5.3-codex", input: 1.75, cachedInput: 0.175, output: 14 },
  { match: "gpt-5.5", input: 5, cachedInput: 0.5, output: 30 },
  { match: "gpt-5.4-mini", input: 0.75, cachedInput: 0.075, output: 4.5 },
  { match: "gpt-5.4-nano", input: 0.2, cachedInput: 0.02, output: 1.25 },
  { match: "gpt-5.4", input: 2.5, cachedInput: 0.25, output: 15 },
];
const CACHE_READ_MULT = 0.1;
const CACHE_WRITE_5M_MULT = 1.25;
const CACHE_WRITE_1H_MULT = 2;

function ratesFor(model) {
  const m = model.toLowerCase();
  return CLAUDE_FAMILY_RATES.find((r) => m.includes(r.match)) ?? null;
}

function openaiRatesFor(model) {
  const m = model.toLowerCase();
  return (
    OPENAI_STANDARD_RATES.find(
      (r) => m === r.match || m.startsWith(`${r.match}-`),
    ) ?? null
  );
}

// Drop a trailing date snapshot (e.g. claude-haiku-4-5-20251001 -> claude-haiku-4-5)
function modelLabel(model) {
  return model.replace(/-\d{8}$/, "");
}

function round2(n) {
  return Math.round(n * 100) / 100;
}

function findJsonlFiles(dir) {
  const out = [];
  let entries;
  try {
    entries = fs.readdirSync(dir, { withFileTypes: true });
  } catch {
    return out;
  }
  for (const entry of entries) {
    const full = path.join(dir, entry.name);
    if (entry.isDirectory()) {
      out.push(...findJsonlFiles(full));
    } else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
      out.push(full);
    }
  }
  return out;
}

function emptyDayModel() {
  return { cost: 0, totalTokens: 0, messageCount: 0 };
}

function emptyDay() {
  return {
    cost: 0,
    totalTokens: 0,
    messageCount: 0,
    byModel: {},
  };
}

function addUsage(state, { timestamp, model, cost, totalTokens }) {
  const ts = timestamp ? Date.parse(timestamp) : NaN;
  if (Number.isNaN(ts)) return;
  if (ts < MIN_USAGE_MS) return;

  if (state.firstMs == null || ts < state.firstMs) state.firstMs = ts;
  if (state.lastMs == null || ts > state.lastMs) state.lastMs = ts;

  const day = new Date(ts).toISOString().slice(0, 10);
  const d = (state.byDay[day] ??= emptyDay());
  d.cost += cost;
  d.totalTokens += totalTokens;
  d.messageCount += 1;

  const dm = (d.byModel[model] ??= emptyDayModel());
  dm.cost += cost;
  dm.totalTokens += totalTokens;
  dm.messageCount += 1;
}

async function processClaudeFile(filePath, state) {
  const rl = readline.createInterface({
    input: fs.createReadStream(filePath),
    crlfDelay: Infinity,
  });

  for await (const line of rl) {
    if (!line) continue;

    let obj;
    try {
      obj = JSON.parse(line);
    } catch {
      continue; // skip malformed lines
    }

    if (obj.type !== "assistant") continue;
    const message = obj.message;
    const usage = message?.usage;
    const model = message?.model;
    if (!usage || !model || model === "<synthetic>") continue;

    // Dedup retries: the same request can be logged more than once.
    const dedupKey =
      obj.requestId && message.id
        ? `${obj.requestId}:${message.id}`
        : obj.uuid;
    if (dedupKey) {
      if (state.seen.has(dedupKey)) continue;
      state.seen.add(dedupKey);
    }

    const input = usage.input_tokens || 0;
    const output = usage.output_tokens || 0;
    const cacheRead = usage.cache_read_input_tokens || 0;

    const breakdown = usage.cache_creation;
    const cc5m = breakdown?.ephemeral_5m_input_tokens;
    const cc1h = breakdown?.ephemeral_1h_input_tokens;
    const cacheCreation =
      cc5m != null || cc1h != null
        ? (cc5m || 0) + (cc1h || 0)
        : usage.cache_creation_input_tokens || 0;

    const rates = ratesFor(model);
    if (!rates) state.unknownModels.add(model);

    let cost = 0;
    if (rates) {
      cost += (input / 1e6) * rates.input;
      cost += (output / 1e6) * rates.output;
      cost += (cacheRead / 1e6) * rates.input * CACHE_READ_MULT;
      if (cc5m != null || cc1h != null) {
        cost += ((cc5m || 0) / 1e6) * rates.input * CACHE_WRITE_5M_MULT;
        cost += ((cc1h || 0) / 1e6) * rates.input * CACHE_WRITE_1H_MULT;
      } else {
        cost +=
          ((usage.cache_creation_input_tokens || 0) / 1e6) *
          rates.input *
          CACHE_WRITE_5M_MULT;
      }
    }

    const lineTokens = input + output + cacheCreation + cacheRead;

    addUsage(state, {
      timestamp: obj.timestamp,
      model: modelLabel(model),
      cost,
      totalTokens: lineTokens,
    });
  }
}

function codexUsageTokens(usage) {
  if (!usage) return 0;
  return (
    usage.total_tokens ??
    (usage.input_tokens || 0) + (usage.output_tokens || 0)
  );
}

function codexCostFor(model, usage, state) {
  const rates = openaiRatesFor(model);
  if (!rates) {
    state.unknownModels.add(`codex:${model}`);
    return 0;
  }

  const input = usage.input_tokens || 0;
  const cachedInput = usage.cached_input_tokens || 0;
  const billableInput = Math.max(0, input - cachedInput);
  const output = usage.output_tokens || 0;

  return (
    (billableInput / 1e6) * rates.input +
    (cachedInput / 1e6) * rates.cachedInput +
    (output / 1e6) * rates.output
  );
}

async function processCodexFile(filePath, state) {
  const rl = readline.createInterface({
    input: fs.createReadStream(filePath),
    crlfDelay: Infinity,
  });

  let currentModel = null;

  for await (const line of rl) {
    if (!line) continue;

    let obj;
    try {
      obj = JSON.parse(line);
    } catch {
      continue; // skip malformed lines
    }

    if (obj.type === "turn_context" && obj.payload?.model) {
      currentModel = obj.payload.model;
      continue;
    }

    if (obj.type !== "event_msg" || obj.payload?.type !== "token_count") {
      continue;
    }

    const usage = obj.payload.info?.last_token_usage;
    const totalTokens = codexUsageTokens(usage);
    if (!currentModel || totalTokens <= 0) continue;

    const dedupKey = `codex:${filePath}:${obj.timestamp}:${totalTokens}`;
    if (state.seen.has(dedupKey)) continue;
    state.seen.add(dedupKey);

    addUsage(state, {
      timestamp: obj.timestamp,
      model: `codex:${modelLabel(currentModel)}`,
      cost: codexCostFor(currentModel, usage, state),
      totalTokens,
    });
  }
}

// Read whatever history is already saved (the days whose logs may be gone).
function readPrev() {
  let prev;
  try {
    prev = JSON.parse(fs.readFileSync(OUT_FILE, "utf8"));
  } catch {
    return { byDay: {}, firstMs: null, lastMs: null };
  }
  const byDay = {};
  for (const day of prev.daily || []) {
    if (!day.date || day.date < MIN_USAGE_DATE) continue;

    const byModel = {};
    for (const [m, v] of Object.entries(day.byModel || {})) {
      byModel[m] = {
        cost: v.cost ?? 0,
        totalTokens: v.totalTokens ?? 0,
        messageCount: v.messageCount ?? 0,
      };
    }
    byDay[day.date] = {
      date: day.date,
      cost: day.cost ?? 0,
      totalTokens: day.totalTokens ?? 0,
      messageCount: day.messageCount ?? 0,
      byModel,
    };
  }
  const firstMs =
    prev.firstMessageAt && Date.parse(prev.firstMessageAt) >= MIN_USAGE_MS
      ? Date.parse(prev.firstMessageAt)
      : null;

  return {
    byDay,
    firstMs,
    lastMs: prev.lastMessageAt ? Date.parse(prev.lastMessageAt) : null,
  };
}

// Round a freshly computed day for storage.
function roundDay(date, d) {
  const byModel = {};
  for (const [m, v] of Object.entries(d.byModel)) {
    byModel[m] = {
      cost: round2(v.cost),
      totalTokens: v.totalTokens,
      messageCount: v.messageCount,
    };
  }
  return {
    date,
    cost: round2(d.cost),
    totalTokens: d.totalTokens,
    messageCount: d.messageCount,
    byModel,
  };
}

function mergeModelUsage(fresh, prev) {
  if (fresh && prev) {
    if (fresh.messageCount !== prev.messageCount) {
      return fresh.messageCount > prev.messageCount ? fresh : prev;
    }
    return fresh.totalTokens >= prev.totalTokens ? fresh : prev;
  }
  return fresh || prev;
}

function mergeDay(date, fresh, prev) {
  if (!fresh || !prev) return fresh || prev;

  const byModel = {};
  const models = new Set([
    ...Object.keys(fresh.byModel || {}),
    ...Object.keys(prev.byModel || {}),
  ]);
  for (const model of models) {
    byModel[model] = mergeModelUsage(
      fresh.byModel?.[model],
      prev.byModel?.[model],
    );
  }

  let cost = 0;
  let totalTokens = 0;
  let modelMessageCount = 0;
  for (const v of Object.values(byModel)) {
    cost += v.cost ?? 0;
    totalTokens += v.totalTokens ?? 0;
    modelMessageCount += v.messageCount ?? 0;
  }

  return {
    date,
    cost: round2(cost),
    totalTokens,
    messageCount: Math.max(
      modelMessageCount,
      fresh.messageCount ?? 0,
      prev.messageCount ?? 0,
    ),
    byModel,
  };
}

async function main() {
  const claudeFiles = findJsonlFiles(CLAUDE_LOGS_DIR);
  const codexFiles = findJsonlFiles(CODEX_LOGS_DIR);

  const state = {
    seen: new Set(),
    unknownModels: new Set(),
    byDay: {},
    firstMs: null,
    lastMs: null,
  };

  for (const file of claudeFiles) {
    await processClaudeFile(file, state);
  }
  for (const file of codexFiles) {
    await processCodexFile(file, state);
  }

  const fresh = {};
  for (const [date, d] of Object.entries(state.byDay)) {
    fresh[date] = roundDay(date, d);
  }

  const prev = readPrev();

  // Merge: for a day in both, keep the copy with more messages (logs can only
  // lose data to pruning, never gain it after the fact).
  const merged = {};
  const dates = new Set([...Object.keys(fresh), ...Object.keys(prev.byDay)]);
  for (const date of dates) {
    const f = fresh[date];
    const p = prev.byDay[date];
    merged[date] = mergeDay(date, f, p);
  }

  // Preserve the true earliest / latest moment across runs.
  const firstCandidates = [prev.firstMs, state.firstMs].filter((v) => v != null);
  const lastCandidates = [prev.lastMs, state.lastMs].filter((v) => v != null);
  const firstMs = firstCandidates.length ? Math.min(...firstCandidates) : null;
  const lastMs = lastCandidates.length ? Math.max(...lastCandidates) : null;

  // Contiguous daily series from the first day to the last, gaps zero-filled.
  const dayKeys = Object.keys(merged).sort();
  const daily = [];
  if (dayKeys.length) {
    let cursor = new Date(dayKeys[0] + "T00:00:00Z");
    const end = new Date(dayKeys[dayKeys.length - 1] + "T00:00:00Z");
    while (cursor <= end) {
      const key = cursor.toISOString().slice(0, 10);
      daily.push(
        merged[key] ?? {
          date: key,
          cost: 0,
          totalTokens: 0,
          messageCount: 0,
          byModel: {},
        },
      );
      cursor = new Date(cursor.getTime() + 86400000);
    }
  }

  // Everything top-level is derived from the merged daily history.
  let totalCost = 0;
  let totalTokens = 0;
  let messageCount = 0;
  const byModel = {};
  for (const day of daily) {
    totalCost += day.cost;
    totalTokens += day.totalTokens;
    messageCount += day.messageCount;
    for (const [m, v] of Object.entries(day.byModel)) {
      const b = (byModel[m] ??= { cost: 0, totalTokens: 0, messageCount: 0 });
      b.cost += v.cost;
      b.totalTokens += v.totalTokens;
      b.messageCount += v.messageCount;
    }
  }
  for (const m of Object.keys(byModel)) byModel[m].cost = round2(byModel[m].cost);

  const result = {
    totalCost: round2(totalCost),
    totalTokens,
    messageCount,
    byModel,
    daily,
    firstMessageAt: firstMs != null ? new Date(firstMs).toISOString() : null,
    lastMessageAt: lastMs != null ? new Date(lastMs).toISOString() : null,
    updatedAt: new Date().toISOString(),
  };

  if (state.unknownModels.size > 0) {
    console.warn(
      `Warning: no pricing for ${[...state.unknownModels].join(
        ", ",
      )} — tokens counted, cost left at 0.`,
    );
  }

  const freshDays = Object.keys(fresh).length;
  const keptDays = daily.filter((d) => d.messageCount > 0).length;
  console.log(
    `Scanned ${claudeFiles.length} Claude file(s) and ${codexFiles.length} Codex file(s). ${freshDays} day(s) in the logs, ${keptDays} day(s) in history.`,
  );
  console.log(
    `Total: $${result.totalCost.toLocaleString()} across ${result.messageCount.toLocaleString()} messages.`,
  );
  for (const [label, m] of Object.entries(byModel).sort(
    (a, b) => b[1].cost - a[1].cost,
  )) {
    console.log(
      `  ${label.padEnd(24)} $${m.cost.toLocaleString().padStart(12)}  ${m.totalTokens.toLocaleString()} tokens`,
    );
  }

  if (DRY_RUN) {
    console.log("\n--dry-run: not writing data/token-usage.json");
    return;
  }

  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, JSON.stringify(result, null, 2) + "\n");
  console.log(`\nWrote ${path.relative(REPO_ROOT, OUT_FILE)}`);
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});