scan.cs

public IEnumerable<Token> Scan(IEnumerable<char> source)
{
    var currentState = StatesTable.InitialStage;
    var currentValue = "";

    Func<Token> markPartialStop = () =>
    {
        if (currentState.IsStop)
        {
            var result = new Token(currentState.ResultingTokenId, currentValue);
            currentState = StatesTable.InitialState;
            currentValue = "";
            return result;
        }
        else
            throw new ArgumentException("Invalid source", "source");
    };

    foreach (var symbol in source)
    {
        if (currentState.Accepts(symbol))
        {
            currentState = StatesTable.Accept(currentState.ChangeTo(symbol));
            currentValue += c;
        }
        else yield return markPartialStop();
    }
    yield return markPartialStop();
}