DataVaultTokenizer

DataVaultTokenizer.cs

using System.Collections.Generic;

using Superpower.Model;
using Superpower.Parsers;
using Superpower;
using System;

namespace datavault.dsl {

    class DataVaultTokenizer : Tokenizer<DataVaultToken> {
        
        protected override IEnumerable<Result<DataVaultToken>> Tokenize(TextSpan span)
        {
            var next = SkipWhiteSpacesAndComments(span);
            if (!next.HasValue)
                yield break;
            do
            {
                // vaild tokens ....

                next = SkipWhiteSpacesAndComments(next.Location);
            } while (next.HasValue);
        }

        private Result<char> SkipWhiteSpacesAndComments(TextSpan span) 
        {
            var next = SkipWhiteSpace(span);
            next = SkipComments(next);
            return SkipWhiteSpace(next.Location);
        }

        private Result<char> SkipComments(Result<char> result) 
        {
            var next = SkipSingleLineComment(result);
            next = SkipWhiteSpace(next.Location);
            return SkipMultiLineComment(next);
        }

        private Result<char> SkipSingleLineComment(Result<char> result) 
        {
            if (result.HasValue && result.Value == '/') {
                var next = result.Remainder.ConsumeChar();
                if (next.HasValue && next.Value == '/') {
                    while(next.HasValue && (next.Value != '\n'))
                        next = next.Remainder.ConsumeChar();
                    return next;
                }
            }
            return result;
        }
        private Result<char> SkipMultiLineComment(Result<char> result) 
        {
            if (result.HasValue && result.Value == '/') {
                var next = result.Remainder.ConsumeChar();
                if (next.HasValue && next.Value == '*') {
                    bool foundAsterix = false;
                    while(next.HasValue && !(foundAsterix && next.Value == '/'))
                    {
                        foundAsterix = next.Value == '*';
                        next = next.Remainder.ConsumeChar();
                    }
                    if (next.HasValue)
                        next = next.Remainder.ConsumeChar();
                    return next;
                }
            }            
            return result;
        }
    }
}