Skip to content

Instantly share code, notes, and snippets.

@7sharp9
Last active Jan 2, 2016
Embed
What would you like to do?
tokenizer snippets
let idents tokens =
//reverse the tokens
let tokens = tokens |> Array.rev |> List.ofArray
let rec loop tokens result =
match tokens with
| [] -> result
| (name, token: TokenInformation) :: tail ->
match token.TokenName with
| "IDENT" -> loop tail (name :: result)
| "DOT" -> loop tail result
| "WHITESPACE" -> loop tail result
| "COMMA" -> loop tail result
| "LPAREN" -> loop tail result
| "RPAREN" -> loop tail result
| _ -> result
loop tokens []
let getIdentIslands tokens =
let tok = tokens |> Array.rev |> List.ofArray
let rec loop tokens current result =
match tokens with
| [] -> match result, current with
| [],_ -> [current] //if the result is empty but we have a 'current', return that.
| _,[] -> result
| _,_ -> (current::result)
| (name, token: TokenInformation) as head::tail ->
match token.TokenName with
| "IDENT" -> loop tail (head::current) result
| "DOT" -> loop tail current result
| "WHITESPACE" -> loop tail current result
| _ -> match current with
| [] -> loop tail [] result
| _ -> loop tail [] (current :: result)//accumulate the current result with the runing result.
loop tok [] []
let getIslandWithinCursor islands cursor =
islands |> List.find
(fun xs -> let filtered =
xs |> List.filter (fun (name: string, token: TokenInformation) ->
cursor >= token.LeftColumn && cursor <= token.RightColumn)
match filtered with
| [] -> false
| _ -> true)
let getIdentAndResidue(doc:Mono.TextEditor.TextDocument) lineStr =
let lines = doc.Lines
|> Seq.map (fun line -> line.ToString())
|> Seq.toList
let currentLineTokens = getLineTokens (doc.FileName.ToString()) lines lineStr
let currentLineEndsWithDot = lineStr.TrimEnd().EndsWith(".")
let ids = idents currentLineTokens
match ids with
| H::[] -> if currentLineEndsWithDot then Some("",[H])
else Some(H,[])
| H::T -> if currentLineEndsWithDot then Some("", ids)
else match ids |> List.rev with
| H::T -> Some(H,T)
| _ -> failwith "Dunno"
| [] -> None
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment