count-tokens: a shell command to count tokens of piped text (using OpenAI's tokenizer). PowerShell and Bash variants.

count-tokens.ps1

# PowerShell: count tokens (GPT o200k_base / GPT-4o+) of piped text.
# Add to your $PROFILE, then: rg 'some-term' src | count-tokens
#
# Requires `uv` (https://docs.astral.sh/uv/). The first run downloads tiktoken
# and its BPE ranks file, then both are cached for fast, offline reuse.
#
# Note: `$input | Out-String` joins pipeline lines with `\r\n` and appends a
# trailing newline, so the count includes a token or two of line-ending
# overhead. For "how big is this for an LLM" purposes that's the right ballpark.

function count-tokens
{
  ($input | Out-String) | uv run --quiet --with tiktoken python -c "import sys,tiktoken; print(len(tiktoken.get_encoding('o200k_base').encode(sys.stdin.buffer.read().decode('utf-8'))))"
}

count-tokens.sh

# Bash/Zsh: count tokens (GPT o200k_base / GPT-4o+) of piped text.
# Add to your ~/.bashrc or ~/.zshrc, then: rg 'some-term' src | count-tokens
#
# Requires `uv` (https://docs.astral.sh/uv/). The first run downloads tiktoken
# and its BPE ranks file, then both are cached for fast, offline reuse.

count-tokens() {
  uv run --quiet --with tiktoken python -c "import sys,tiktoken; print(len(tiktoken.get_encoding('o200k_base').encode(sys.stdin.buffer.read().decode('utf-8'))))"
}