better-tokenizer.cs

public void Tokenize(string value, List<(int Start, int Length)> tokens)
{
    // To find a word we wait for:
    // 1. a char that is noice
    // 2. a buffer larger than zero that is not a stopword

    int length = 0,  start = 0;
    for (int i = 0; i < value.Length; i++)
    {
        var ch = char.ToLowerInvariant(value[i]);
        if (!IsNoice(ch))
        {
            length++;
            continue;
        }
        if (length == 0)
        {
            start++;
            continue;
        }
        if (IsStopword(value, start, length) == false)
        {
            tokens.Add((start, length));
            start += length + 1;
            length = 0;
        }
    }

    if (length > 0 && IsStopword(value, start, length) == false)
    {
        tokens.Add((start, length));
    }
}