tiktoken-example

main.mjs

import { encoding_for_model } from "tiktoken";

const enc = encoding_for_model("gpt-5");

const tokens = enc.encode("A świstak siedzi i zawija je w te sreberka");
console.log(tokens);

const dec = [];
tokens.forEach((t) => {
  const decoded = enc.decode([t]);
  const decodedStr = String.fromCharCode.apply(null, decoded);
  dec.push(decodedStr);
});

console.log(dec);

enc.free();