Tokenizer sample

TokenizerSample.cs

using Microsoft.DeepDev;

namespace OpenAISdkSamples;

public class TokenizerSample
{
    private const string ImStart = "<|im_start|>";
    private const string ImEnd = "<|im_end|>";

    private static readonly Dictionary<string, int> SpecialTokens = new()
    {
        { ImStart,100264},
        { ImEnd, 100265},
    };

    private static readonly IReadOnlyCollection<string> AllowedSpecial = new HashSet<string>(SpecialTokens.Keys);

    public async Task RunAsync()
    {
        var text = "こんにちは、猫さん";

        var count = await CountTokenAsync(text, "gpt-3.5-turbo");
        Console.WriteLine(count);
    }

    public async Task<int> CountTokenAsync(string text, string modelName)
    {
        var tokenizer = await TokenizerBuilder.CreateByModelNameAsync(modelName, SpecialTokens);
        var input = $"{ImStart}{text}{ImEnd}";
        var encoded = tokenizer.Encode(input, AllowedSpecial);
        return encoded.Count - 2;
    }
}