Quick and dirty GPT Tokenizer in one command

README.md

dev

deno run --watch --allow-net main.ts
curl -d 'test world :)' http://localhost:8085

prod

deno run --allow-net=0.0.0.0:8085 https://gist.githubusercontent.com/JLarky/6a77d9d483b7f0067fd3eea1b785e628/raw/3cf707ae447f8309564b94613a014f9ddf3f972d/main.ts

curl -d 'test world :)' http://localhost:8085

main.ts

import { Application } from "https://deno.land/x/oak@v11.1.0/mod.ts";
import GPT3Tokenizer from "https://esm.sh/gpt3-tokenizer@1.1.5";

const app = new Application();

app.use(async (ctx) => {
  if (ctx.request.method === "POST" && ctx.request.url.pathname === "/") {
    // read the body
    const body = ctx.request.body({ type: "text" });
    const value = await body.value;
    const encoded = tokenizer.encode(value);

    // respond with json
    ctx.response.body = encoded;
  } else {
    ctx.response.body = "Hello World!";
  }
});
const tokenizer = new GPT3Tokenizer({ type: "gpt3" }); // or 'codex'

console.log("Server started on port http://localhost:8085");
await app.listen({ port: 8085 });