-
-
Save heitor-lassarote/c8f12feab735439a50015b7c60d4f21a to your computer and use it in GitHub Desktop.
Lexer.x: Identifiers
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
module Lexer | |
( -- * Invoking Alex | |
Alex | |
, AlexPosn (..) | |
, alexGetInput | |
, alexError | |
, runAlex | |
, alexMonadScan | |
, Range (..) | |
, RangedToken (..) | |
, Token (..) | |
, scanMany | |
) where | |
import Data.ByteString.Lazy.Char8 (ByteString) | |
import qualified Data.ByteString.Lazy.Char8 as BS | |
} | |
%wrapper "monadUserState-bytestring" | |
$digit = [0-9] | |
$alpha = [a-zA-Z] | |
@id = ($alpha | \_) ($alpha | $digit | \_ | \' | \?)* | |
tokens :- | |
<0> $white+ ; | |
<0> @id { tokId } | |
{ | |
data AlexUserState = AlexUserState | |
{ | |
} | |
alexInitUserState :: AlexUserState | |
alexInitUserState = AlexUserState | |
alexEOF :: Alex RangedToken | |
alexEOF = do | |
(pos, _, _, _) <- alexGetInput | |
pure $ RangedToken EOF (Range pos pos) | |
data Range = Range | |
{ start :: AlexPosn | |
, stop :: AlexPosn | |
} deriving (Eq, Show) | |
data RangedToken = RangedToken | |
{ rtToken :: Token | |
, rtRange :: Range | |
} deriving (Eq, Show) | |
data Token | |
-- Identifiers | |
= Identifier ByteString | |
-- Constants | |
| String ByteString | |
| Integer Integer | |
-- Keywords | |
| Let | |
| In | |
| If | |
| Then | |
| Else | |
-- Arithmetic operators | |
| Plus | |
| Minus | |
| Times | |
| Divide | |
-- Comparison operators | |
| Eq | |
| Neq | |
| Lt | |
| Le | |
| Gt | |
| Ge | |
-- Logical operators | |
| And | |
| Or | |
-- Parenthesis | |
| LPar | |
| RPar | |
-- Lists | |
| Comma | |
| LBrack | |
| RBrack | |
-- Types | |
| Colon | |
| Arrow | |
-- EOF | |
| EOF | |
deriving (Eq, Show) | |
mkRange :: AlexInput -> Int64 -> Range | |
mkRange (start, _, str, _) len = Range{start = start, stop = stop} | |
where | |
stop = BS.foldl' alexMove start $ BS.take len str | |
tokId :: AlexAction RangedToken | |
tokId inp@(_, _, str, _) len = | |
pure RangedToken | |
{ rtToken = Identifier $ BS.take len str | |
, rtRange = mkRange inp len | |
} | |
scanMany :: ByteString -> Either String [RangedToken] | |
scanMany input = runAlex input go | |
where | |
go = do | |
output <- alexMonadScan | |
if rtToken output == EOF | |
then pure [output] | |
else (output :) <$> go | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment