Lexer Api

lexer.fs

module Lexer =

    open System.Threading
    open FSharp.Compiler.UnicodeLexing
    open FSharp.Compiler.Range
    open FSharp.Compiler.Ast
    open FSharp.Compiler.Text
    open FSharp.Compiler.Features
    open FSharp.Compiler.Parser
    open FSharp.Compiler.Lexhelp
    open Internal.Utilities

    [<Flags>]
    type FSharpLexerFlags =
        | None                          = 0x00000
        | Default                       = 0x11010
        | LightSyntaxOff                = 0x00001
        | Compiling                     = 0x00010 
        | CompilingFSharpCore           = 0x00110
        | SkipTrivia                    = 0x01000
        | UseLexFilter                  = 0x10000

    [<Struct;RequireQualifiedAccess>]
    type FSharpSyntaxTokenKind =
        | None
        | Text
        | Keyword
        | Identifier
        | StringLiteral
        | NumericLiteral
        | Comment

    [<Struct;NoComparison;NoEquality>]
    type FSharpSyntaxToken(tok: token, range: range) =

        member _.Range = range

        member _.Kind =
            match tok with
            | _ when obj.ReferenceEquals(tok, null) -> FSharpSyntaxTokenKind.None
            | Parser.token.ABSTRACT
            | Parser.token.AND
            | Parser.token.AS
            | Parser.token.ASSERT
            | Parser.token.BASE
            | Parser.token.BEGIN
            | Parser.token.CLASS
            | Parser.token.DEFAULT
            | Parser.token.DELEGATE
            | Parser.token.DO
            | Parser.token.DONE
            | Parser.token.DOWNCAST
            | Parser.token.DOWNTO
            | Parser.token.ELIF
            | Parser.token.ELSE
            | Parser.token.END
            | Parser.token.EXCEPTION
            | Parser.token.EXTERN
            | Parser.token.FALSE
            | Parser.token.FINALLY
            | Parser.token.FIXED
            | Parser.token.FOR
            | Parser.token.FUN
            | Parser.token.FUNCTION
            | Parser.token.GLOBAL
            | Parser.token.IF
            | Parser.token.IN
            | Parser.token.INHERIT
            | Parser.token.INLINE
            | Parser.token.INTERFACE
            | Parser.token.INTERNAL
            | Parser.token.LAZY
            | Parser.token.LET _ // "let" and "use"
            | Parser.token.DO_BANG //  "let!", "use!" and "do!"
            | Parser.token.MATCH
            | Parser.token.MATCH_BANG
            | Parser.token.MEMBER
            | Parser.token.MODULE
            | Parser.token.MUTABLE
             
            | Parser.token.NAMESPACE
            | Parser.token.NEW
            // | Parser.token.NOT // Not actually a keyword. However, not struct in combination is used as a generic parameter constraint.
            | Parser.token.NULL
            | Parser.token.OF
            | Parser.token.OPEN
            | Parser.token.OR
            | Parser.token.OVERRIDE
            | Parser.token.PRIVATE
            | Parser.token.PUBLIC
            | Parser.token.REC
            | Parser.token.YIELD _ // "yield" and "return"
            | Parser.token.YIELD_BANG _ // "yield!" and "return!"
            | Parser.token.STATIC
            | Parser.token.STRUCT
            | Parser.token.THEN
            | Parser.token.TO
            | Parser.token.TRUE
            | Parser.token.TRY
            | Parser.token.TYPE
            | Parser.token.UPCAST
            | Parser.token.VAL
            | Parser.token.VOID
            | Parser.token.WHEN
            | Parser.token.WHILE
            | Parser.token.WITH

            // * Reserved - from OCAML *
            | Parser.token.ASR
            | Parser.token.INFIX_STAR_STAR_OP "asr"
            | Parser.token.INFIX_STAR_DIV_MOD_OP "land"
            | Parser.token.INFIX_STAR_DIV_MOD_OP "lor"
            | Parser.token.INFIX_STAR_STAR_OP "lsl"
            | Parser.token.INFIX_STAR_STAR_OP "lsr"
            | Parser.token.INFIX_STAR_DIV_MOD_OP "lxor"
            | Parser.token.INFIX_STAR_DIV_MOD_OP "mod"
            | Parser.token.SIG

            // * Reserved - for future *
            // atomic
            // break
            // checked
            // component
            // const
            // constraint
            // constructor
            // continue
            // eager
            // event
            // external
            // functor
            // include
            // method
            // mixin
            // object
            // parallel
            // process
            // protected
            // pure
            // sealed
            // tailcall
            // trait
            // virtual
            // volatile
            | Parser.token.RESERVED
            | Parser.token.KEYWORD_STRING _ -> FSharpSyntaxTokenKind.Keyword
            | Parser.token.IDENT _ -> FSharpSyntaxTokenKind.Identifier
            | Parser.token.STRING _ -> FSharpSyntaxTokenKind.StringLiteral
            | Parser.token.UINT8 _
            | Parser.token.INT16 _
            | Parser.token.INT32 _
            | Parser.token.INT64 _
            | Parser.token.BIGNUM _ -> FSharpSyntaxTokenKind.NumericLiteral
            | Parser.token.COMMENT _ -> FSharpSyntaxTokenKind.Comment
            | Parser.token.LINE_COMMENT _ -> FSharpSyntaxTokenKind.Comment
            | _ -> FSharpSyntaxTokenKind.Text

    let lexWithErrorLogger (text: ISourceText) (filePath: string) conditionalCompilationDefines (flags: FSharpLexerFlags) supportsFeature errorLogger lexCallback pathMap (ct: CancellationToken) =
        let canSkipTrivia = (flags &&& FSharpLexerFlags.SkipTrivia) = FSharpLexerFlags.SkipTrivia
        let isLightSyntaxOn = (flags &&& FSharpLexerFlags.LightSyntaxOff) <> FSharpLexerFlags.LightSyntaxOff
        let isCompiling = (flags &&& FSharpLexerFlags.Compiling) = FSharpLexerFlags.Compiling
        let isCompilingFSharpCore = (flags &&& FSharpLexerFlags.CompilingFSharpCore) = FSharpLexerFlags.CompilingFSharpCore
        let canUseLexFilter = (flags &&& FSharpLexerFlags.UseLexFilter) = FSharpLexerFlags.UseLexFilter

        let lexbuf = UnicodeLexing.SourceTextAsLexbuf(supportsFeature, text)
        let lightSyntaxStatus = LightSyntaxStatus(isLightSyntaxOn, true) 
        let lexargs = mkLexargs (filePath, conditionalCompilationDefines, lightSyntaxStatus, Lexhelp.LexResourceManager 0, [], errorLogger, pathMap)
        let lexargs = { lexargs with applyLineDirectives = not isCompiling }

        let getNextToken =
            let lexer = Lexer.token lexargs canSkipTrivia

            if canUseLexFilter then
                fun lexbuf ->
                    let tokenizer = LexFilter.LexFilter(lexargs.lightSyntaxStatus, isCompilingFSharpCore, lexer, lexbuf)
                    tokenizer.Lexer lexbuf
            else
                lexer

        usingLexbufForParsing (lexbuf, filePath) (fun lexbuf -> 
            lexCallback lexbuf (fun lexbuf -> ct.ThrowIfCancellationRequested (); getNextToken lexbuf))

    let lex text filePath conditionalCompilationDefines flags supportsFeature lexCallback pathMap ct =
        let errorLogger = CompilationErrorLogger("Lexer", ErrorLogger.FSharpErrorSeverityOptions.Default)
        lexWithErrorLogger text filePath conditionalCompilationDefines flags supportsFeature errorLogger lexCallback pathMap ct

    [<AbstractClass;Sealed>]
    type FSharpLexer =

        static member Lex(text: ISourceText, tokenCallback, ?langVersion, ?filePath, ?conditionalCompilationDefines, ?flags, ?pathMap, ?ct) =
            let langVersion = defaultArg langVersion "latestmajor"
            let flags = defaultArg flags FSharpLexerFlags.Default
            let filePath = defaultArg filePath String.Empty
            let conditionalCompilationDefines = defaultArg conditionalCompilationDefines []
            let pathMap = defaultArg pathMap Map.Empty
            let ct = defaultArg ct CancellationToken.None

            let supportsFeature = (LanguageVersion langVersion).SupportsFeature

            let pathMap =
                (PathMap.empty, pathMap)
                ||> Seq.fold (fun state pair -> state |> PathMap.addMapping pair.Key pair.Value)

            let lexCallback =
                fun (lexbuf: Lexbuf) getNextToken ->
                    while not lexbuf.IsPastEndOfStream do
                        tokenCallback (FSharpSyntaxToken(getNextToken lexbuf, lexbuf.LexemeRange))

            lex text filePath conditionalCompilationDefines flags supportsFeature lexCallback pathMap ct