Tokenizer.ml

#light

module Token

let rec private readToken inQuote tokenChars restChars =
    let chars2Token chars = new string(chars |> Array.ofList |> Array.rev)
    if inQuote then
        match restChars with
        | '\'' :: [] -> (chars2Token tokenChars, [])
        | '\'' :: '-' :: cs -> (chars2Token tokenChars, '-' :: cs)
        | '\'' :: '\'' :: cs -> readToken true ('\'' :: tokenChars) cs
        | c :: cs -> readToken true (c :: tokenChars) cs
        | _ -> 
            let rest = new string(restChars |> Array.ofList);
            failwith ("expect an close quote but failed. rest: " + rest)
    else
        match restChars with
        | []
        | '-' :: _  -> (chars2Token tokenChars, restChars)
        | '\'' :: _ -> failwith "start quote unexpected."
        | c :: cs -> readToken false (c :: tokenChars) cs

let rec private readTokenGroup tokens chars = 
    match chars with
    | []
    | '-' :: '-' :: _ -> (tokens |> List.rev, chars)
    | '-' :: cs -> readTokenGroup tokens cs
    | '\'' :: cs -> 
        let (t, rest) = readToken true [] cs
        readTokenGroup (t :: tokens) rest
    | _ ->
        let (t, rest) = readToken false [] chars
        readTokenGroup (t :: tokens) rest
        
let parse (text : string) =
    let rec parse' groups chars = 
        match chars with
        | [] -> groups |> List.rev
        | '-' :: '-' :: cs -> parse' groups cs
        | _ ->
            let (g, rest) = readTokenGroup [] chars
            parse' (g :: groups) rest
    text |> List.ofSeq |> parse' []