itertokens.h

#pragma once

// String tokenizing iterator
// by Nathan Reed, 2020-08-06. Licensed CC0 https://creativecommons.org/publicdomain/zero/1.0/
//
// Use it like:
//
// for (auto token : IterTokens(" Bird \t\tFish Dog   Cat "))
// {
//     // token is a std::string_view pointing into the original string
//     std::cout << "Token: " << token << "\n";
// }
//
// The input string is not modified.
// Both the input string and the delimiter string must remain valid at least till the end of the loop.
// (This could be made more sophisticated so that it could take other string types as input,
// auto-capture temporaries, etc., but I haven't done that here.)
// Godbolt example: https://godbolt.org/z/8PnjrW

constexpr auto IterTokens(const char* str, const char* delimiters = "\t ")
{
    struct iterator
    {
        const char* tokenStart;
        const char* tokenEnd;
        const char* delimiters;
        bool operator != (const iterator & other) const { return tokenEnd != other.tokenEnd; }
        bool isDelimiter(char c) { return strchr(delimiters, c) != nullptr; }
        void next()
        {
            if (!tokenEnd) return;

            // Find the beginning of the next token
            tokenStart = tokenEnd;
            while (*tokenStart != 0 && isDelimiter(*tokenStart)) ++tokenStart;
            if (*tokenStart == 0) { tokenEnd = nullptr; return; }

            // Find the end of the token
            tokenEnd = tokenStart;
            while (*tokenEnd != 0 && !isDelimiter(*tokenEnd)) ++tokenEnd;
        }
        iterator& operator ++ () { next(); return *this; }
        std::string_view operator * () const
        {
            return tokenEnd ? std::string_view(tokenStart, size_t(tokenEnd - tokenStart)) : std::string_view();
        }
    };
    struct iterable
    {
        const char* str;
        const char* delimiters;
        auto begin() { return ++iterator{str, str, delimiters}; }
        auto end() { return iterator{nullptr, nullptr, nullptr}; }
    };
    return iterable{str, delimiters};
}