mockup_tokenizer.jl

module Mockup

abstract type TokenType end
struct AToken <: TokenType end
struct BToken <: TokenType end

struct Token{T <: TokenType}
    position::Int
    value::Char
end

scan(c::Char, input) = first(input) == c

scan(::AToken, input) = scan('A', input)
scan(::BToken, input) = scan('B', input)

const REGISTERED_TOKENTYPES = [AToken(), BToken()]

function scan(input; charnum=0)
    success = false
    tokentype = nothing
    for scanner in REGISTERED_TOKENTYPES
        success = scan(scanner, input)
        if success
            tokentype = scanner
            break
        end
    end
    if !success
        error("No suitable token found for input at char $charnum")
    end
    Token{typeof(tokentype)}(charnum, first(input))
end

tokenize(input) = Token[scan(c, charnum=i) for (i,c) in enumerate(input)]

end